oneDAL development env docker image (oneapi-src#2513)

* removing JDK deps * Fixing env and introduce docker * Creating bazel env script * adding readme for docker * updating * adding github validation * new line * remove bash invocation for scripts * fixing ci.yml * add execute permissions * fixing ci.yml * fixing github flow name * adding sudo to docker image * fixing install dpcpp script * fixing base docker deps * fixing bazelisk * fixing ci.yml * fixing windows ci with bash call * fixing bazel ci * updating bazel readme to current state * fixing bazel caching and nuget windows build * adding git binary * adding build validation and conda to env * enable larger validation in CI * fixing CLI * adding conda to docker * fixing docker * fixing docker * adding deps and build * adding docs env and more tests for nightly * not passing param * switching to conda env file * adding conda env * fixign path to env.yml * drop caching version to 1 as bazel version was fixed * bump version for spxinks-prompt to 1.8 * revert version * disabling doc reqs * switchign to latest tag * actually running code in docker * merging docker commands in one * merging docker commands in one * merging docker commands in one * installing bazel in docker image * adding build with OpenBLAS to validation * removing build with OpenBLAS to validation * Apply suggestions from code review Co-authored-by: Alexandra <alexandra.epanchinzeva@intel.com> * Apply suggestions from code review Co-authored-by: Alexander Andreev <alexander.andreev@intel.com> --------- Co-authored-by: Alexandra <alexandra.epanchinzeva@intel.com> Co-authored-by: Alexander Andreev <alexander.andreev@intel.com>
avolkov-intel · Oct 4, 2023 · 9188113 · 9188113
1 parent 744f259
commit 9188113
Show file tree

Hide file tree

Showing 12 changed files with 247 additions and 64 deletions.
diff --git a/.ci/env/oneapi.sh → .ci/env/apt.sh b/.ci/env/oneapi.sh → .ci/env/apt.sh
@@ -17,6 +17,10 @@
 
 component=$1
 
+function update {
+    sudo apt-get update
+}
+
 function add_repo {
     wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
     sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
@@ -27,14 +31,7 @@ function add_repo {
 }
 
 function install_dpcpp {
-    sudo apt-get install                    \
-        intel-oneapi-common-vars            \
-        intel-oneapi-common-licensing       \
-        intel-oneapi-tbb-devel              \
-        intel-oneapi-dpcpp-cpp-compiler     \
-        intel-oneapi-dev-utilities          \
-        intel-oneapi-libdpstd-devel         \
-        cmake
+    sudo apt-get install -y intel-dpcpp-cpp-compiler-2023.2.1
     sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd'
     sudo mv -f /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga_
 }
@@ -43,14 +40,35 @@ function install_mkl {
     sudo apt-get install intel-oneapi-mkl-devel
 }
 
+function install_clang-format {
+    sudo apt-get install -y clang-format-14
+    sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-14 100
+    sudo update-alternatives --set clang-format /usr/bin/clang-format-14
+}
+
+function install_dev-base {
+    sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree
+}
+
+function install_dev-base-conda {
+    conda env create -f .ci/env/environment.yml
+}
+
 if [ "${component}" == "dpcpp" ]; then
     add_repo
     install_dpcpp
 elif [ "${component}" == "mkl" ]; then
     add_repo
     install_mkl
+elif [ "${component}" == "clang-format" ]; then
+    update
+    install_clang-format
+elif [ "${component}" == "dev-base" ]; then
+    update
+    install_dev-base
+    install_dev-base-conda
 else
     echo "Usage:"
-    echo "   $0 [dpcpp|mkl]"
+    echo "   $0 [dpcpp|mkl|clang-format|dev-base]"
     exit 1
 fi
diff --git a/.ci/scripts/install_dpcpp.sh → .ci/env/bazelisk.sh b/.ci/scripts/install_dpcpp.sh → .ci/env/bazelisk.sh
@@ -15,12 +15,13 @@
 # limitations under the License.
 #===============================================================================
 
-wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
-sudo add-apt-repository -y "deb https://apt.repos.intel.com/oneapi all main"
-sudo apt-get update
-sudo apt-get install -y intel-dpcpp-cpp-compiler-2023.2.1
-sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd'
-sudo mv -f /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga_
+# Download Bazelisk
+export SHA256="ce52caa51ef9e509fb6b7e5ad892e5cf10feb0794b0aed4d2f36adb00a1a2779  bazelisk-linux-amd64"
+wget https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64
+echo ${SHA256} | sha256sum --check
+# "Install" bazelisk
+chmod +x bazelisk-linux-amd64
+mkdir -p bazel/bin
+mv bazelisk-linux-amd64 bazel/bin/bazel
+export BAZEL_VERSION=$(./bazel/bin/bazel --version | awk '{print $2}')
+export PATH=$PATH:$(pwd)/bazel/bin
diff --git a/.ci/env/environment.yml b/.ci/env/environment.yml
@@ -0,0 +1,7 @@
+name: ci-env
+channels:
+  - conda-forge
+  - intel
+  - defaults
+dependencies:
+  - impi-devel=2021.10.0
diff --git a/.ci/env/tbb.bat b/.ci/env/tbb.bat
diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml
@@ -28,9 +28,7 @@ jobs:
     vmImage: 'ubuntu-22.04'
   steps:
     - script: |
-        sudo apt-get update && sudo apt-get install -y clang-format-14
-        sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-14 100
-        sudo update-alternatives --set clang-format /usr/bin/clang-format-14
+        .ci/env/apt.sh clang-format
       displayName: 'apt-get'
     - script: |
         .ci/scripts/clang-format.sh
@@ -46,8 +44,7 @@ jobs:
     vmImage: 'ubuntu-22.04'
   steps:
   - script: |
-      sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree
-      conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.10.0 openjdk=17.0.3
+      .ci/env/apt.sh dev-base
     displayName: 'apt-get and conda install'
   - script: |
       .ci/scripts/describe_system.sh
@@ -95,8 +92,7 @@ jobs:
     vmImage: 'ubuntu-22.04'
   steps:
   - script: |
-      sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree
-      conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.8.0 openjdk=17.0.3
+      .ci/env/apt.sh dev-base
     displayName: 'apt-get and conda install'
   - script: |
       .ci/scripts/describe_system.sh
@@ -136,11 +132,10 @@ jobs:
     vmImage: 'ubuntu-22.04'
   steps:
   - script: |
-      sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix
-      conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 openjdk=17.0.3
+      .ci/env/apt.sh dev-base
     displayName: 'apt-get and conda install'
   - script: |
-      bash .ci/scripts/install_dpcpp.sh
+      .ci/env/apt.sh dpcpp
     displayName: 'dpcpp installation'
   - script: |
       source /opt/intel/oneapi/compiler/latest/env/vars.sh
@@ -190,27 +185,21 @@ jobs:
     BAZEL_CACHE_MAX_SIZE_KB: 4194304 # Size in kilobytes ~ 4Gb
   steps:
   - script: |
-      # Download Bazelisk
-      export SHA256="168851e70cf5f95c0e215e7f3aaca5132ffc3c8dd8f585a4157b0be2b53cfe32 bazelisk-linux-amd64"
-      wget https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64
-      echo ${SHA256} | sha256sum --check
-      # "Install" bazelisk
-      chmod +x bazelisk-linux-amd64
-      mv bazelisk-linux-amd64 ${BAZEL_VERSION}
-
-    displayName: 'install-deps'
+      # sourcing done to set bazel version value from script
+      source .ci/env/bazelisk.sh
+      echo "##vso[task.setvariable variable=BAZEL_VERSION]${BAZEL_VERSION}"
+    displayName: 'install-bazel'
   - script: |
       .ci/scripts/describe_system.sh
     displayName: 'System info'
-
   - task: Cache@2
     inputs:
       # Commit ID is added to a cache key. Caches are immutable by design,
       # so we always need to change a key to upload the last version
       # of the Bazel cache. Cache lookup is based on `restoreKeys` option.
-      key: '"$(BAZEL_VERSION)" | "$(Agent.OS)" | "v2" | "$(Build.SourceVersion)"'
+      key: '"$(BAZEL_VERSION)" | "$(Agent.OS)" | "v1" | "$(Build.SourceVersion)"'
       restoreKeys: |
-        "$(BAZEL_VERSION)" | "$(Agent.OS)" | "v2"
+        "$(BAZEL_VERSION)" | "$(Agent.OS)" | "v1"
       path: $(BAZEL_CACHE_DIR)
     displayName: 'bazel-cache'
 
@@ -227,41 +216,41 @@ jobs:
       echo "build --disk_cache=$(BAZEL_CACHE_DIR) --cpu=avx2" > ~/.bazelrc
 
       # Display oneDAL build configuration
-      $(BAZEL_VERSION) build @config//:dump
+      bazel build @config//:dump
       echo
       cat bazel-bin/external/config/config.json
       echo
     displayName: 'bazel-configure'
   - script: |
-      $(BAZEL_VERSION) build :release
+      bazel build :release
     displayName: 'release'
   - task: PublishPipelineArtifact@1
     inputs:
       artifactName: '$(platform.type) Bazel build'
       targetPath: '$(Build.Repository.LocalPath)/bazel-bin/release'
     displayName: 'Upload build artifacts'
   - script: |
-      $(BAZEL_VERSION) test //examples/oneapi/cpp:all \
+      bazel test //examples/oneapi/cpp:all \
                  --test_link_mode=dev \
                  --test_thread_mode=par
     displayName: 'cpp-examples-thread-dev'
 
   - script: |
       export DALROOT=`pwd`/bazel-bin/release/daal/latest
-      $(BAZEL_VERSION) test //examples/oneapi/cpp:all \
+      bazel test //examples/oneapi/cpp:all \
                  --test_link_mode=release_static \
                  --test_thread_mode=par
     displayName: 'cpp-examples-thread-release-static'
 
   - script: |
       export DALROOT=`pwd`/bazel-bin/release/daal/latest
-      $(BAZEL_VERSION) test //examples/oneapi/cpp:all \
+      bazel test //examples/oneapi/cpp:all \
                  --test_link_mode=release_dynamic \
                  --test_thread_mode=par
     displayName: 'cpp-examples-thread-release-dynamic'
 
   - script: |
-      $(BAZEL_VERSION) test //cpp/oneapi/dal:tests \
+      bazel test //cpp/oneapi/dal:tests \
                  --config=host \
                  --test_link_mode=dev \
                  --test_thread_mode=par
@@ -310,7 +299,7 @@ jobs:
     inputs:
       artifact: '$(platform.type) build'
       path: $(Pipeline.Workspace)
-  - script: sudo apt-get update && sudo apt-get install -y git gcc-multilib g++-multilib
+  - script: .ci/env/apt.sh dev-base
     displayName: 'apt-get'
   - script: |
       .ci/scripts/describe_system.sh
@@ -392,7 +381,7 @@ jobs:
   steps:
   - script: |
       brew install dos2unix tree
-      conda create -n ci-env -q -y -c conda-forge python=3.9 openjdk=17.0.3
+      conda create -n ci-env -q -y -c conda-forge python=3.10
       source /usr/local/miniconda/etc/profile.d/conda.sh
       conda activate ci-env
       pip install -q cpufeature

diff --git a/.github/workflows/docker-validation-ci.yml b/.github/workflows/docker-validation-ci.yml
@@ -0,0 +1,19 @@
+name: docker-validation CI
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  validate:
+    name: Docker validation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+      - name: Build docker image
+        run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest
diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml
@@ -0,0 +1,35 @@
+name: docker-validation Nightly
+
+on:
+  schedule:
+    - cron: "0 23 * * *"
+  pull_request:
+    branches:
+      - master
+    paths:
+      - .github/workflows/docker-validation-nightly.yml
+      - dev/docker/onedal-dev.Dockerfile
+      - .ci/env
+      - .ci/scripts
+  push:
+    branches:
+      - master
+    paths:
+      - .github/workflows/docker-validation-nightly.yml
+      - dev/docker/onedal-dev.Dockerfile
+      - .ci/env
+      - .ci/scripts
+
+jobs:
+  validate:
+    name: Docker validation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+      - name: Build docker image
+        run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest
+      - name: Building oneDAL
+        run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env
+      - name: Building oneDAL with bazel
+        run: docker run onedal-dev bazel build :release
diff --git a/INSTALL.md b/INSTALL.md
@@ -28,6 +28,11 @@ Required Software:
 
 For details, see [System Requirements for oneDAL](https://www.intel.com/content/www/us/en/developer/articles/system-requirements/system-requirements-for-oneapi-data-analytics-library.html).
 
+## Docker Development Environment 
+
+[Docker file](https://github.com/oneapi-src/oneDAL/tree/master/dev/docker) with the oneDAL development environment
+is available as an alternative to the manual setup. 
+
 ## Installation Steps
 1. Clone the sources from GitHub\* as follows:
 

diff --git a/dev/bazel/README.md b/dev/bazel/README.md
@@ -1,17 +1,33 @@
+<!--
+******************************************************************************
+* Copyright 2023 Intel Corporation
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*******************************************************************************/-->
 # Bazel Guide
 ## Install Bazel on Linux
-1. Download Bazel 4.0.0
+1. Download Bazelisk
    ```sh
-   wget -O bazel https://github.com/bazelbuild/bazel/releases/download/4.0.0/bazel-4.0.0-linux-x86_64
+   wget -O bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64
    ```
    > Note: If you are using proxy don't forget to set
    `http_proxy` and `https_proxy` environment variables
 
 2. Put it somewhere on **local disk** (use of remote shared
    drives is not recommended) and set executable attributes.
    ```sh
-   mv bazel <my-user-dir-utils>/bin
    chmod +x bazel
+   mv bazel <my-user-dir-utils>/bin
    ```
 
 3. Create Bazel cache directory on **local disk** and
@@ -24,7 +40,7 @@
 4. Add `bazel` to the `$PATH`.
    ```sh
    export PATH=<my-user-dir-on-local-disk>/bin:$PATH
-   bazel --version # Should be "bazel 4.0.0"
+   bazel --version
    ```
 
 ### Compiler choice
@@ -197,10 +213,6 @@ The most used Bazel commands are `build`, `test` and `run`.
    bazel test --test_link_mode=release_dynamic //cpp/oneapi/dal:tests
    ```
 
-- `--test_thread_mode` Specifies threading mode for tests. \
-  Possible values:
-  - `par` _default_ Links against `onedal_thread`.
-
    Example:
    ```sh
    bazel test --test_thread_mode=par //cpp/oneapi/dal:tests