SynapseAi 1.15.0 release

 * Update dockerfiles with 1.15.0 content

dockerfiles/README.md

@@ -30,24 +30,23 @@ This script can be used as reference to build docker images for Gaudi.
 1. Go into the folder of the image type you would like to build:
     * base
     * pytorch
-    * tensorflow
     * triton
 
 2. Run build command to generate Docker image
     ```
     make build
     ```
     Examples:
-    #### Build pytorch image for ubuntu22.04:
+    #### Build pytorch image for rhel9.2:
     ```
     cd pytorch
-    make build BUILD_OS=ubuntu22.04
+    make build BUILD_OS=rhel9.2
     ```
 
-    #### Build tensorflow image rhel8.6:
+    #### Build triton image (default OS - ubuntu22.04):
     ```
-    cd tensorflow
-    make build BUILD_OS=rhel8.6
+    cd triton
+    make build
     ```
 
 3. Build command variables

dockerfiles/base/Dockerfile.amzn2

@@ -57,19 +57,18 @@ RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
     echo "baseurl=https://${ARTIFACTORY_URL}/artifactory/AmazonLinux2" >> /etc/yum.repos.d/habanalabs.repo && \
     echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/AmazonLinux2/repodata/repomd.xml.key" >> /etc/yum.repos.d/habanalabs.repo
 
-RUN yum install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".amzn2 && \
+RUN yum makecache && \
+    yum install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".amzn2 && \
     yum install -y habanalabs-thunk-"$VERSION"-"$REVISION".amzn2 && \
     yum install -y habanalabs-firmware-tools-"$VERSION"-"$REVISION".amzn2 && \
-    yum install -y habanalabs-graph-"$VERSION"-"$REVISION".amzn2
-
-RUN rpm -V habanalabs-rdma-core && rpm -V habanalabs-thunk && rpm -V habanalabs-firmware-tools && rpm -V habanalabs-graph
+    yum install -y habanalabs-graph-"$VERSION"-"$REVISION".amzn2 && \
+    rpm -V habanalabs-rdma-core && rpm -V habanalabs-thunk && rpm -V habanalabs-firmware-tools && rpm -V habanalabs-graph && \
+    rm -f /etc/yum.repos.d/habanalabs.repo && \
+    yum clean all && rm -rf /var/cache/yum
 
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 && \
     update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
 
-RUN rm -f /etc/yum.repos.d/habanalabs.repo && \
-    yum clean all && rm -rf /var/cache/yum
-
 # SSH configuration necessary to support mpi-operator v2
 RUN sed -i 's/[ #]\(.*StrictHostKeyChecking \).*/ \1no/g' /etc/ssh/ssh_config && \
     sed -i 's/[ #]\(.*ForwardAgent \).*/ \1yes/g' /etc/ssh/ssh_config && \
@@ -88,13 +87,12 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
     ./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
     make && make install
 
-ENV LIBOFI_VERSION="0.0.1"
-RUN wget -nv -O /tmp/v${LIBOFI_VERSION}.tar.gz https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/tags/v${LIBOFI_VERSION}.tar.gz && \
-    cd /tmp/ && tar xf /tmp/v${LIBOFI_VERSION}.tar.gz && \
-    cd /tmp/hccl_ofi_wrapper-${LIBOFI_VERSION} && \
+RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
+    unzip /tmp/main.zip -d /tmp && \
+    cd /tmp/hccl_ofi_wrapper-main && \
     make && cp -f libhccl_ofi_wrapper.so /usr/lib/habanalabs/libhccl_ofi_wrapper.so && \
     cd / && \
-    rm -rf /tmp/v${LIBOFI_VERSION}.tar.gz /tmp/hccl_ofi_wrapper-${LIBOFI_VERSION}
+    rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
 RUN python3 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
 

dockerfiles/base/Dockerfile.debian10.10

@@ -75,23 +75,12 @@ ENV RDMA_CORE_LIB=${RDMA_CORE_ROOT}/build/lib
 ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH
 ENV PATH=${LIBFABRIC_ROOT}/bin:$PATH
 
-# ucx installation
-ENV UCX_VERSION="1.15.0"
-ENV UCX_ROOT="/opt/habanalabs/ucx-${UCX_VERSION}"
-RUN wget -nv -O /tmp/ucx-${UCX_VERSION}.tar.gz https://github.com/openucx/ucx/releases/download/v${UCX_VERSION}/ucx-${UCX_VERSION}.tar.gz && \
-    cd /tmp/ && tar xf /tmp/ucx-${UCX_VERSION}.tar.gz && \
-    cd /tmp/ucx-${UCX_VERSION} && \
-    ./configure --prefix=${UCX_ROOT} && \
-    make && make install
-ENV LD_LIBRARY_PATH=${UCX_ROOT}/lib:$LD_LIBRARY_PATH
-ENV PATH=${UCX_ROOT}/bin:$PATH
-
 # install openmpi
 ENV MPI_ROOT="/usr/local/share/openmpi"
 RUN wget -nv -P /tmp/openmpi "https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VER}.tar.gz" && \
     cd /tmp/openmpi && tar xzf openmpi-${OPENMPI_VER}.tar.gz && \
     cd /tmp/openmpi/openmpi-${OPENMPI_VER} && \
-    ./configure --prefix=${MPI_ROOT} --with-sge --disable-builtin-atomics --enable-orterun-prefix-by-default --with-ucx=${UCX_ROOT} --with-verbs && \
+    ./configure --prefix=${MPI_ROOT} --with-sge --disable-builtin-atomics --enable-orterun-prefix-by-default --with-verbs && \
     make -j 8 && \
     sudo make install && \
     cp LICENSE ${MPI_ROOT} && \
@@ -149,13 +138,12 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
     ./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
     make && make install
 
-ENV LIBOFI_VERSION="0.0.1"
-RUN wget -nv -O /tmp/v${LIBOFI_VERSION}.tar.gz https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/tags/v${LIBOFI_VERSION}.tar.gz && \
-    cd /tmp/ && tar xf /tmp/v${LIBOFI_VERSION}.tar.gz && \
-    cd /tmp/hccl_ofi_wrapper-${LIBOFI_VERSION} && \
+RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
+    unzip /tmp/main.zip -d /tmp && \
+    cd /tmp/hccl_ofi_wrapper-main && \
     make && cp -f libhccl_ofi_wrapper.so /usr/lib/habanalabs/libhccl_ofi_wrapper.so && \
     cd / && \
-    rm -rf /tmp/v${LIBOFI_VERSION}.tar.gz /tmp/hccl_ofi_wrapper-${LIBOFI_VERSION}
+    rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.8 2 && \
     update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1

dockerfiles/base/Dockerfile.rhel8.6 → dockerfiles/base/Dockerfile.rhel9.2

@@ -1,9 +1,9 @@
-# Copyright (c) 2023 Habana Labs, Ltd.
+# Copyright (c) 2024 Habana Labs, Ltd.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-# HabanaLabs Dockerfile base installer layer for RedHat 8.6
-FROM registry.access.redhat.com/ubi8/ubi:8.6
+# HabanaLabs Dockerfile base installer layer for RedHat 9.2
+FROM registry.access.redhat.com/ubi9/ubi:9.2
 ARG ARTIFACTORY_URL
 ARG VERSION
 ARG REVISION
@@ -13,20 +13,19 @@ LABEL release="${VERSION}-${REVISION}"
 
 COPY LICENSE /licenses/
 
-RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
     dnf clean all && rm -rf /var/cache/yum
 
-RUN echo "[appstream]" > /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
-    echo "name=CentOS Linux 8 - AppStream" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
-    echo "mirrorlist=http://mirrorlist.centos.org/?release=\$releasever-stream&arch=\$basearch&repo=AppStream&infra=\$infra" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
-    echo "gpgcheck=0" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo
-
-
 RUN echo "[BaseOS]" > /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
-    echo "name=CentOS Linux 8 - BaseOS" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
-    echo "mirrorlist=http://mirrorlist.centos.org/?release=\$releasever-stream&arch=\$basearch&repo=BaseOS&infra=\$infra" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
+    echo "name=CentOS Linux 9 - BaseOS" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
+    echo "baseurl=https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
     echo "gpgcheck=0" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
 
+RUN echo "[centos9]" > /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
+    echo "name=CentOS Linux 9 - AppStream" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
+    echo "baseurl=https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo
+
 RUN dnf install -y \
     clang \
     cmake3 \
@@ -42,13 +41,18 @@ RUN dnf install -y \
     unzip \
     llvm \
     lsof \
-    python38-devel \
+    python3-devel \
     openssh-clients \
+    openssl \
+    openssl-devel \
     libjpeg-devel \
     openssh-server \
-    redhat-lsb-core \
+    lsb_release \
     wget \
     git \
+    libffi-devel \
+    bzip2-devel \
+    zlib-devel \
     mesa-libGL \
     iproute \
     python3-dnf-plugin-versionlock && \
@@ -57,6 +61,10 @@ RUN dnf install -y \
     dnf update -y && \
     dnf clean all && rm -rf /var/cache/yum
 
+COPY install-python310.sh .
+RUN ./install-python310.sh rhel9.2 && rm install-python310.sh
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+
 COPY install_efa.sh .
 RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh
 
@@ -71,20 +79,18 @@ ENV RDMAV_FORK_SAFE=1
 ENV FI_EFA_USE_DEVICE_RDMA=1
 
 RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
-    echo "name=Habana RH8 Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \
-    echo "baseurl=https://${ARTIFACTORY_URL}/artifactory/rhel/8/8.6" >> /etc/yum.repos.d/habanalabs.repo && \
-    echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/rhel/8/8.6/repodata/repomd.xml.key" >> /etc/yum.repos.d/habanalabs.repo
-
-RUN echo "[powertools]" > /etc/yum.repos.d/powertools.repo && \
-    echo "name=powertools" >> /etc/yum.repos.d/powertools.repo && \
-    echo "baseurl=http://mirror.centos.org/centos/8-stream/PowerTools/x86_64/os/"  >> /etc/yum.repos.d/powertools.repo && \
-    echo "gpgcheck=0" >> /etc/yum.repos.d/powertools.repo
-
-RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el8 \
-        habanalabs-thunk-"$VERSION"-"$REVISION".el8 \
-        habanalabs-firmware-tools-"$VERSION"-"$REVISION".el8 \
-        habanalabs-graph-"$VERSION"-"$REVISION".el8 && \
-    rm -f /etc/yum.repos.d/habanalabs.repo && rm -rf /tmp/* && \
+    echo "name=Habana RH9 Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \
+    echo "baseurl=https://${ARTIFACTORY_URL}/artifactory/rhel/9/9.2" >> /etc/yum.repos.d/habanalabs.repo && \
+    echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/rhel/9/9.2/repodata/repomd.xml.key" >> /etc/yum.repos.d/habanalabs.repo && \
+    echo 'gpgcheck=1' >> /etc/yum.repos.d/habanalabs.repo
+
+Run update-crypto-policies --set LEGACY
+
+RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el9 \
+    habanalabs-thunk-"$VERSION"-"$REVISION".el9 \
+    habanalabs-firmware-tools-"$VERSION"-"$REVISION".el9 \
+    habanalabs-graph-"$VERSION"-"$REVISION".el9 && \
+    rm -f /etc/yum.repos.d/habanalabs.repo && rm -f /etc/yum.repos.d/habana.repo && rm -rf /tmp/* && \
     dnf clean all && rm -rf /var/cache/yum
 
 RUN rpm -V habanalabs-rdma-core && rpm -V habanalabs-thunk && rpm -V habanalabs-firmware-tools && rpm -V habanalabs-graph
@@ -101,21 +107,18 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
     ./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
     make && make install
 
-ENV LIBOFI_VERSION="0.0.1"
-RUN wget -nv -O /tmp/v${LIBOFI_VERSION}.tar.gz https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/tags/v${LIBOFI_VERSION}.tar.gz && \
-    cd /tmp/ && tar xf /tmp/v${LIBOFI_VERSION}.tar.gz && \
-    cd /tmp/hccl_ofi_wrapper-${LIBOFI_VERSION} && \
+RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
+    unzip /tmp/main.zip -d /tmp && \
+    cd /tmp/hccl_ofi_wrapper-main && \
     make && cp -f libhccl_ofi_wrapper.so /usr/lib/habanalabs/libhccl_ofi_wrapper.so && \
     cd / && \
-    rm -rf /tmp/v${LIBOFI_VERSION}.tar.gz /tmp/hccl_ofi_wrapper-${LIBOFI_VERSION}
+    rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
-RUN python3.8 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
+RUN python3.10 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
 
-RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 && \
-    alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 && \
-    alternatives --set python3 /usr/bin/python3.8
+RUN ln -s /usr/bin/python3 /usr/bin/python
 
-RUN python3.8 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
+RUN python3.10 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
 
 # SSH configuration necessary to support mpi-operator v2
 RUN mkdir -p /var/run/sshd && \

dockerfiles/base/Dockerfile.tencentos3.1

@@ -84,13 +84,12 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
     ./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
     make && make install
 
-ENV LIBOFI_VERSION="0.0.1"
-RUN wget -nv -O /tmp/v${LIBOFI_VERSION}.tar.gz https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/tags/v${LIBOFI_VERSION}.tar.gz && \
-    cd /tmp/ && tar xf /tmp/v${LIBOFI_VERSION}.tar.gz && \
-    cd /tmp/hccl_ofi_wrapper-${LIBOFI_VERSION} && \
+RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
+    unzip /tmp/main.zip -d /tmp && \
+    cd /tmp/hccl_ofi_wrapper-main && \
     make && cp -f libhccl_ofi_wrapper.so /usr/lib/habanalabs/libhccl_ofi_wrapper.so && \
     cd / && \
-    rm -rf /tmp/v${LIBOFI_VERSION}.tar.gz /tmp/hccl_ofi_wrapper-${LIBOFI_VERSION}
+    rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
 RUN python3.8 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
 

dockerfiles/base/Dockerfile.ubuntu22.04

@@ -97,13 +97,12 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
     ./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
     make && make install
 
-ENV LIBOFI_VERSION="0.0.1"
-RUN wget -nv -O /tmp/v${LIBOFI_VERSION}.tar.gz https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/tags/v${LIBOFI_VERSION}.tar.gz && \
-    cd /tmp/ && tar xf /tmp/v${LIBOFI_VERSION}.tar.gz && \
-    cd /tmp/hccl_ofi_wrapper-${LIBOFI_VERSION} && \
+RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
+    unzip /tmp/main.zip -d /tmp && \
+    cd /tmp/hccl_ofi_wrapper-main && \
     make && cp -f libhccl_ofi_wrapper.so /usr/lib/habanalabs/libhccl_ofi_wrapper.so && \
     cd / && \
-    rm -rf /tmp/v${LIBOFI_VERSION}.tar.gz /tmp/hccl_ofi_wrapper-${LIBOFI_VERSION}
+    rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
 RUN python3 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
 

dockerfiles/tensorflow/install-python310.sh → dockerfiles/base/install-python310.sh

@@ -10,12 +10,12 @@ case "${_BASE_NAME}" in
         echo "Skip install Python3.10 from source on Ubuntu22.04"
         exit 0;
     ;;
-    *debian*)
+    *debian* | *ubuntu*)
         apt update
         apt install -y libsqlite3-dev libreadline-dev
     ;;
     *rhel*)
-        yum install -y sqlite-devel readline-devel
+        yum install -y sqlite-devel readline-devel xz-devel
     ;;
     *amzn2*)
         yum install -y sqlite-devel readline-devel
@@ -45,11 +45,11 @@ make -j && make altinstall
 
 # post install
 case "${_BASE_NAME}" in
-    *rhel8*)
-        alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.10 3 && \
-        alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 && \
-        alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 && \
+    *rhel9*)
+        alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.10 2 && \
+        alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 && \
         alternatives --set python3 /usr/local/bin/python3.10
+        export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
     ;;
     *amzn2*)
         update-alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.10 3 && \

dockerfiles/common.mk

@@ -5,10 +5,9 @@ BUILD_OS ?= ubuntu22.04
 BUILD_DIR ?= $(CURDIR)/dockerbuild
 
 REPO_SERVER ?= vault.habana.ai
-TF_VERSION ?= 2.15.0
-PT_VERSION ?= 2.1.1
-RELEASE_VERSION ?= 1.14.0
-RELEASE_BUILD_ID ?= 493
+PT_VERSION ?= 2.2.0
+RELEASE_VERSION ?= 1.15.0
+RELEASE_BUILD_ID ?= 479
 
 BASE_IMAGE_URL ?= base-installer-$(BUILD_OS)
 IMAGE_URL = $(IMAGE_NAME):$(RELEASE_VERSION)-$(RELEASE_BUILD_ID)

dockerfiles/pytorch/Dockerfile.amzn2

@@ -28,7 +28,9 @@ RUN yum install -y \
     zlib-devel \
     lapack-devel \
     openblas-devel \
-    numactl && \
+    pdsh \
+    numactl \
+    libmkl-dev && \
     yum clean all
 
 RUN amazon-linux-extras install epel -y

dockerfiles/pytorch/Dockerfile.debian10.10

@@ -29,9 +29,16 @@ RUN apt update && apt install -y \
     libpcre2-dev \
     libselinux1-dev \
     moreutils \
+    pdsh \
     numactl && \
     apt clean
 
+RUN apt update && apt install -y software-properties-common && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 3 && \
+    apt-add-repository -y non-free && apt update && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 && \
+    apt install -y libmkl-dev
+
 # Default python is pointing to 3.7
 RUN bash -c "\
     ln -sf /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 /lib/x86_64-linux-gnu/libtcmalloc.so.4; \