SynapseAi 0.15.1 release

* Modify paths to 0.15.1-37 * Update v0.15.1 dockerfile changes
HabanaAI · Jul 6, 2021 · 71fc19f · 71fc19f
1 parent c14da4b
commit 71fc19f
Show file tree

Hide file tree

Showing 10 changed files with 168 additions and 177 deletions.
diff --git a/README.md b/README.md
diff --git a/dockerfiles/Dockerfile_amzn2_pytorch_installer b/dockerfiles/Dockerfile_amzn2_pytorch_installer
@@ -17,6 +17,7 @@ ENV LANG=en_US.UTF-8
 RUN yum install -y \
     unzip \
     openssh-clients \
+    openssh-server \
     curl \
     redhat-lsb-core \
     openmpi-devel \

diff --git a/dockerfiles/Dockerfile_amzn2_tensorflow_installer b/dockerfiles/Dockerfile_amzn2_tensorflow_installer
@@ -13,6 +13,7 @@ ARG REVISION
 ARG TF_VERSION=2.2.2
 ARG OPENMPI_VER=4.0.5
 ARG ARTIFACTORY_URL
+
 ENV TF_MODULES_RELEASE_BUILD=/usr/lib/habanalabs/
 ENV PYTHONPATH=/root:/usr/lib/habanalabs/
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/habanalabs/openmpi/lib/
@@ -45,31 +46,22 @@ RUN wget --no-verbose https://download.open-mpi.org/release/open-mpi/v4.0/openmp
 ENV CC=/usr/lib/habanalabs/openmpi/bin/mpicc
 ENV MPICC=/usr/lib/habanalabs/openmpi/bin/mpicc
 
+COPY requirements-training-release.txt requirements-training-release.txt
+
 RUN wget https://bootstrap.pypa.io/get-pip.py && \
-    python3 get-pip.py pip==19.3.1 && \
+    python3 get-pip.py pip==21.0.1 && \
     rm -rf get-pip.py && \
-    pip3 install gast==0.2.2 \
-         py-cpuinfo==5.0.0 \
-         requests==2.20 \
-         tensorflow_datasets==1.2.0 \
-         tensorflow-metadata==0.12.1 \
-         cython==0.29.15 \
-         imgaug==0.4.0 \
-         keras==2.3.1 \
-         cloudpickle==1.6.0 \
-         git+https://github.com/NVIDIA/dllogger.git@26a0f8f1958de2c0c460925ff6102a4d2486d6cc#egg=dllogger \
-         numpy==1.19.2 && \
-         # pycocotools has to be installed in separated process otherwise it fails
+    pip3 install -r requirements-training-release.txt && \
+    # pycocotools has to be installed in separated process otherwise it fails
     pip3 install pycocotools==2.0.0 \
          tensorflow-cpu==${TF_VERSION} \
-         tensorflow-model-optimization==0.5.0
+         tensorflow-model-optimization==0.5.0 && \
+    rm requirements-training-release.txt
 
 # Using pip install habana-tensorflow and habana-horovod python packages
 RUN python3 -m pip install habana-tensorflow=="${VERSION}"."${REVISION}" \
         --index-url "https://${ARTIFACTORY_URL}"/api/pypi/gaudi-python/simple && \
     python3 -m pip install habana-horovod=="${VERSION}"."${REVISION}" \
         --index-url "https://${ARTIFACTORY_URL}"/api/pypi/gaudi-python/simple && \
     echo "source /etc/profile.d/habanalabs.sh" >> ~/.bashrc && \
-    echo "/etc/init.d/ssh start \"-p 3022\"" >> ~/.bashrc
-
-RUN pip3 install mpi4py --no-cache-dir
+    echo "/etc/init.d/ssh start \"-p 3022\"" >> ~/.bashrc
diff --git a/dockerfiles/Dockerfile_ubuntu18.04_base_installer b/dockerfiles/Dockerfile_ubuntu18.04_base_installer
@@ -7,6 +7,7 @@ FROM ubuntu:bionic-20210512
 ARG ARTIFACTORY_URL
 ARG VERSION
 ARG REVISION
+ARG HABANA_PIP_VERSION="19.3.1"
 
 ENV LANG=en_US.UTF-8
 ENV LANGUAGE=en_US.UTF-8
@@ -19,50 +20,50 @@ ENV OS_NUMBER=1804
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    apt-transport-https=1.6.13 \
-    apt-utils=1.6.13 \
-    ethtool=1:4.15-0ubuntu1 \
-    gpg-agent=2.2.4-1ubuntu1.4 \
-    libjemalloc1=3.6.0-11 \
-    python3-venv=3.6.7-1~18.04 \
-    software-properties-common=0.96.24.32.14 \
-    wget=1.19.4-1ubuntu2.2 && \
+    apt-transport-https \
+    apt-utils \
+    ethtool \
+    gpg-agent \
+    libjemalloc1 \
+    python3-venv \
+    software-properties-common \
+    wget && \
     apt-get autoremove && apt-get clean
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential=12.4ubuntu1 \
-    ca-certificates=20210119~18.04.1 \
-    dkms=2.3-3ubuntu9.7 \
-    gcc=4:7.4.0-1ubuntu2.3 \
-    graphviz=2.40.1-2 \
-    libgoogle-glog0v5=0.3.5-1 \
-    libopencv-dev=3.2.0+dfsg-4ubuntu0.1 \
-    libpq-dev=10.17-0ubuntu0.18.04.* \
-    locales=2.27-3ubuntu1.4 \
-    make=4.1-9.1ubuntu1 \
-    python3.6=3.6.9-1~18.04ubuntu1.4 \
-    python3.6-dev=3.6.9-1~18.04ubuntu1.4 \
+    build-essential \
+    ca-certificates \
+    dkms \
+    gcc \
+    graphviz \
+    libgoogle-glog0v5 \
+    libopencv-dev \
+    libpq-dev \
+    locales \
+    make \
+    python3.6 \
+    python3.6-dev \
     python3.6-tk \
-    protobuf-compiler=3.0.0-9.1ubuntu1 && \
+    python3-pip \
+    protobuf-compiler && \
     apt-get autoremove && apt-get clean
 
 RUN apt-get update && \
     ### Install unzip to extract pre-trained weights for BERT demo
     ### Install ssh for horovod/MPI
     apt-get install -y --no-install-recommends \
-    bc=1.07.1-2 \
-    git=1:2.17.1-1ubuntu0.8 \
-    openssh-client=1:7.6p1-4ubuntu0.3 \
-    openssh-server=1:7.6p1-4ubuntu0.3 \
-    unzip=6.0-21ubuntu1.1 \
-    vim=2:8.0.1453-1ubuntu1.4 && \
+    bc \
+    git \
+    openssh-client \
+    openssh-server \
+    unzip \
+    vim && \
     apt-get autoremove && apt-get clean
 
 RUN locale-gen en_US.UTF-8
-RUN wget https://bootstrap.pypa.io/get-pip.py && \
-    python3.6 get-pip.py pip==19.3.1 && \
-    rm get-pip.py
-RUN pip3.6 install setuptools==41.0.0
+
+RUN python3 -m pip install pip=="${HABANA_PIP_VERSION}" && \
+    pip3 install setuptools==41.0.0
 
 RUN yes '' | add-apt-repository ppa:deadsnakes/ppa && \
     echo "deb https://${ARTIFACTORY_URL}/debian bionic main" | tee -a /etc/apt/sources.list && \

diff --git a/dockerfiles/Dockerfile_ubuntu18.04_py37_base_installer b/dockerfiles/Dockerfile_ubuntu18.04_py37_base_installer
@@ -7,6 +7,7 @@ FROM ubuntu:bionic-20210512
 ARG ARTIFACTORY_URL
 ARG VERSION
 ARG REVISION
+ARG HABANA_PIP_VERSION="19.3.1"
 
 ENV LANG=en_US.UTF-8
 ENV LANGUAGE=en_US.UTF-8
@@ -19,61 +20,60 @@ ENV OS_NUMBER=1804
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    apt-transport-https=1.6.13 \
-    apt-utils=1.6.13 \
-    ethtool=1:4.15-0ubuntu1 \
-    gpg-agent=2.2.4-1ubuntu1.4 \
-    libjemalloc1=3.6.0-11 \
-    software-properties-common=0.96.24.32.14 \
-    wget=1.19.4-1ubuntu2.2 && \
+    apt-transport-https \
+    apt-utils \
+    ethtool \
+    gpg-agent \
+    libjemalloc1 \
+    software-properties-common \
+    wget && \
     apt-get autoremove && apt-get clean
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential=12.4ubuntu1 \
-    ca-certificates=20210119~18.04.1 \
-    dkms=2.3-3ubuntu9.7 \
-    gcc=4:7.4.0-1ubuntu2.3 \
-    graphviz=2.40.1-2 \
-    libgoogle-glog0v5=0.3.5-1 \
-    libopencv-dev=3.2.0+dfsg-4ubuntu0.1 \
-    libpq-dev=10.17-0ubuntu0.18.04.1 \
-    locales=2.27-3ubuntu1.4 \
-    make=4.1-9.1ubuntu1 \
-    protobuf-compiler=3.0.0-9.1ubuntu1 && \
+    build-essential \
+    ca-certificates \
+    dkms \
+    gcc \
+    graphviz \
+    libgoogle-glog0v5 \
+    libopencv-dev \
+    libpq-dev \
+    locales \
+    make \
+    protobuf-compiler && \
     apt-get autoremove && apt-get clean
 
 RUN apt-get update && \
     ### Install ssh for horovod/MPI
     apt-get install -y --no-install-recommends \
-    bc=1.07.1-2 \
-    git=1:2.17.1-1ubuntu0.8 \
-    openssh-client=1:7.6p1-4ubuntu0.3 \
-    openssh-server=1:7.6p1-4ubuntu0.3 \
-    unzip=6.0-21ubuntu1.1 \
-    vim=2:8.0.1453-1ubuntu1.4 && \
+    bc \
+    git \
+    openssh-client \
+    openssh-server \
+    unzip \
+    vim \
+    python3-pip && \
     apt-get autoremove && apt-get clean
 
 RUN locale-gen en_US.UTF-8
 # Install python 3.7 as default version
 RUN apt-get update && \
     # Install python 3.7 packages
     apt-get install -y --no-install-recommends \
-        python3.7=3.7.5-2~18.04.* \
-        python3.7-dev=3.7.5-2~18.04.* \
-        python3.7-venv=3.7.5-2~18.04.* \
+        python3.7 \
+        python3.7-dev \
+        python3.7-venv \
         # there is no dedicated package distutils, tk for python 3.7
-        python3-distutils=3.6.9-1~18.04 \
-        python3-tk=3.6.9-1~18.04 && \
+        python3-distutils \
+        python3-tk && \
     apt-get autoremove --yes && apt-get clean && rm -rf /var/lib/apt/lists/* && \
     # Configure alternatives
         update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 10 && \
         update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 && \
         update-alternatives --install /usr/bin/python python /usr/bin/python3.7 10 && \
-        update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 && \
-    # Install pip package manager and setup tools
-        wget -q https://bootstrap.pypa.io/get-pip.py && \
-        python3 get-pip.py pip==19.3.1 && \
-        rm get-pip.py && \
+        update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1
+
+RUN python3 -m pip install pip=="${HABANA_PIP_VERSION}" && \
         pip3 install setuptools==41.0.0
 
 # Required symlinks for fix cannot import modues _gi, apt_pkg after python 3.7 installation in add-apt-repository script

diff --git a/dockerfiles/Dockerfile_ubuntu20.04_base_installer b/dockerfiles/Dockerfile_ubuntu20.04_base_installer
@@ -19,42 +19,42 @@ ENV OS_NUMBER=2004
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    apt-transport-https=2.0.5 \
-    apt-utils=2.0.5 \
-    ethtool=1:5.4-1 \
-    gpg-agent=2.2.19-3ubuntu2.1 \
-    libjemalloc2=5.2.1-1ubuntu1 \
-    python3-venv=3.8.2-0ubuntu2 \
-    software-properties-common=0.98.9.* \
-    wget=1.20.3-1ubuntu1 && \
+    apt-transport-https \
+    apt-utils \
+    ethtool \
+    gpg-agent \
+    libjemalloc2 \
+    python3-venv \
+    software-properties-common \
+    wget && \
     apt-get autoremove && apt-get clean
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential=12.8ubuntu1.1 \
-    ca-certificates=20210119~20.04.1 \
-    dkms=2.8.1-5ubuntu1 \
-    gcc=4:9.3.0-1ubuntu2 \
-    graphviz=2.42.2-3build2 \
-    libgoogle-glog0v5=0.4.0-1build1 \
-    libopencv-dev=4.2.0+dfsg-5 \
-    libpq-dev=12.7-0ubuntu0.20.04.* \
-    locales=2.31-0ubuntu9.2 \
-    make=4.2.1-1.2 \
-    python3=3.8.2-0ubuntu2 \
-    python3-dev=3.8.2-0ubuntu2 \
-    python3-tk=3.8.10-0ubuntu1~20.04 \
-    python3-pip=20.0.2-5ubuntu1.* \
-    protobuf-compiler=3.6.1.3-2ubuntu5 && \
+    build-essential \
+    ca-certificates \
+    dkms \
+    gcc \
+    graphviz \
+    libgoogle-glog0v5 \
+    libopencv-dev \
+    libpq-dev \
+    locales \
+    make \
+    python3 \
+    python3-dev \
+    python3-tk \
+    python3-pip \
+    protobuf-compiler && \
     apt-get autoremove && apt-get clean
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    bc=1.07.1-2build1 \
-    git=1:2.25.1-1ubuntu3 \
-    openssh-client=1:8.2p1-4ubuntu0.* \
-    openssh-server=1:8.2p1-4ubuntu0.* \
-    unzip=6.0-25ubuntu1 \
-    vim=2:8.1.2269-1ubuntu5 && \
+    bc \
+    git \
+    openssh-client \
+    openssh-server \
+    unzip \
+    vim && \
     apt-get autoremove && apt-get clean
 
 RUN locale-gen en_US.UTF-8

diff --git a/dockerfiles/Dockerfile_ubuntu_pytorch_installer b/dockerfiles/Dockerfile_ubuntu_pytorch_installer
@@ -12,6 +12,7 @@ ARG REVISION
 ARG BASE_NAME
 ARG ARTIFACTORY_URL
 ARG OPENMPI_VER=4.0.5
+ARG HABANA_PIP_VERSION="19.3.1"
 
 ENV LANG=en_US.UTF-8
 
@@ -41,11 +42,9 @@ RUN bash -c "\
         apt-get clean; \
     fi"
 
-RUN curl --create-dirs -sSLo get-pip.py https://bootstrap.pypa.io/get-pip.py && \
-    python get-pip.py pip==19.3.1 --no-warn-script-location && \
-    rm -rf get-pip.py
+RUN python3 -m pip install pip=="${HABANA_PIP_VERSION}"
 
-# Install openmpi version 4.0.5 (SW-42854)
+# Install openmpi version 4.0.5
 RUN wget --no-verbose https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-"${OPENMPI_VER}".tar.gz && \
     tar -xvf openmpi-"${OPENMPI_VER}".tar.gz && \
     cd openmpi-"${OPENMPI_VER}" && \

diff --git a/dockerfiles/Dockerfile_ubuntu_tensorflow_installer b/dockerfiles/Dockerfile_ubuntu_tensorflow_installer
@@ -38,13 +38,11 @@ RUN wget --no-verbose https://download.open-mpi.org/release/open-mpi/v4.0/openmp
     cd - && \
     rm -rf openmpi-"${OPENMPI_VER}"* && \
     /sbin/ldconfig
-RUN wget --no-verbose https://bootstrap.pypa.io/get-pip.py && \
-    python3 get-pip.py pip==21.0.1 && \
-    rm -rf get-pip.py
 
 COPY requirements-training-release.txt requirements-training-release.txt
 
-RUN pip3 install -r requirements-training-release.txt && \
+RUN python3 -m pip install pip==21.0.1 && \
+    pip3 install -r requirements-training-release.txt && \
     pip3 uninstall --yes habana tensorflow && \
     # tensorflow-cpu and -model have to be installed in separated processes otherwise old version of tf will be imported
     pip3 install tensorflow-cpu==${TF_VERSION} && \

diff --git a/dockerfiles/docker_build.sh b/dockerfiles/docker_build.sh
@@ -6,12 +6,12 @@
 #
 # HabanaLabs script for building docker images
 
-: "${1?"Usage: $0 mode [base,tensorflow,pytorch] os [ubuntu18.04,ubuntu20.04,amzn2] tf_version [2.2.2, 2.4.1]"}"
-: "${2?"Usage: $0 mode [base,tensorflow,pytorch] os [ubuntu18.04,ubuntu20.04,amzn2] tf_version [2.2.2, 2.4.1]"}"
-: "${2?"Usage: $0 mode [base,tensorflow,pytorch] os [ubuntu18.04,ubuntu20.04,amzn2] tf_version [2.2.2, 2.4.1]"}"
+: "${1?"Usage: $0 mode [base,tensorflow,pytorch] os [ubuntu18.04,ubuntu20.04,amzn2] tf_version [2.4.1, 2.5.0]"}"
+: "${2?"Usage: $0 mode [base,tensorflow,pytorch] os [ubuntu18.04,ubuntu20.04,amzn2] tf_version [2.4.1, 2.5.0]"}"
+: "${2?"Usage: $0 mode [base,tensorflow,pytorch] os [ubuntu18.04,ubuntu20.04,amzn2] tf_version [2.4.1, 2.5.0]"}"
 
-VERSION="${CUSTOM_VERSION:-0.15.0}"
-REVISION="${CUSTOM_REVISION:-547}"
+VERSION="${CUSTOM_VERSION:-0.15.1}"
+REVISION="${CUSTOM_REVISION:-37}"
 MODE="$1"
 OS="$2"
 TF_VERSION="$3"
@@ -93,7 +93,7 @@ function buildDocker {
            if [[ "$TF_VERSION" == "2.2."* || "$TF_VERSION" == "2.4."* || "$TF_VERSION" == "2.5."* ]]; then
                TF_CPU_POSTFIX="-tf-cpu-${TF_VERSION}"
            else
-               TF_VERSION="2.4.1"  # Set tensorflow-cpu==2.4.1
+               TF_VERSION="2.5.0"  # Set tensorflow-cpu==2.5.0
            fi
            BUILDARGS+=" --build-arg ARTIFACTORY_URL="$ARTIFACTORY_URL" --build-arg TF_VERSION="$TF_VERSION" --build-arg VERSION="$VERSION" --build-arg REVISION="$REVISION""
          ;;

diff --git a/dockerfiles/requirements-training-release.txt b/dockerfiles/requirements-training-release.txt
@@ -39,7 +39,7 @@ mpi4py==3.0.3
     pypng==0.0.20
     httplib2==0.19.0
     google-auth-httplib2==0.0.4
-    urllib3==1.26.4
+    urllib3>=1.26.5
     uritemplate==3.0.1
     oauth2client==4.1.3
     google-api-core==1.25.0