oneDAL development env docker image

.ci/env/oneapi.sh → .ci/env/apt.sh

@@ -17,6 +17,10 @@
 
 component=$1
 
+function update {
+ sudo apt-get update
+}
+
 function add_repo {
  wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
  sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
@@ -27,14 +31,7 @@ function add_repo {
 }
 
 function install_dpcpp {
- sudo apt-get install \
- intel-oneapi-common-vars \
- intel-oneapi-common-licensing \
- intel-oneapi-tbb-devel \
- intel-oneapi-dpcpp-cpp-compiler \
- intel-oneapi-dev-utilities \
- intel-oneapi-libdpstd-devel \
- cmake
+ sudo apt-get install -y intel-dpcpp-cpp-compiler-2023.2.1
  sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd'
  sudo mv -f /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga_
 }
@@ -43,14 +40,35 @@ function install_mkl {
  sudo apt-get install intel-oneapi-mkl-devel
 }
 
+function install_clang-format {
+ sudo apt-get install -y clang-format-14
+ sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-14 100
+ sudo update-alternatives --set clang-format /usr/bin/clang-format-14
+}
+
+function install_dev-base {
+ sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree
+}
+
+function install_dev-base-conda {
+ conda env create -f .ci/env/environment.yml
+}
+
 if [ "${component}" == "dpcpp" ]; then
  add_repo
  install_dpcpp
 elif [ "${component}" == "mkl" ]; then
  add_repo
  install_mkl
+elif [ "${component}" == "clang-format" ]; then
+ update
+ install_clang-format
+elif [ "${component}" == "dev-base" ]; then
+ update
+ install_dev-base
+ install_dev-base-conda
 else
  echo "Usage:"
- echo " $0 [dpcpp|mkl]"
+ echo " $0 [dpcpp|mkl|clang-format|dev-base]"
  exit 1
 fi

.ci/scripts/install_dpcpp.sh → .ci/env/bazelisk.sh

@@ -15,12 +15,13 @@
 # limitations under the License.
 #===============================================================================
 
-wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
-sudo add-apt-repository -y "deb https://apt.repos.intel.com/oneapi all main"
-sudo apt-get update
-sudo apt-get install -y intel-dpcpp-cpp-compiler-2023.2.1
-sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd'
-sudo mv -f /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga_
+# Download Bazelisk
+export SHA256="ce52caa51ef9e509fb6b7e5ad892e5cf10feb0794b0aed4d2f36adb00a1a2779 bazelisk-linux-amd64"
+wget https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64
+echo ${SHA256} | sha256sum --check
+# "Install" bazelisk
+chmod +x bazelisk-linux-amd64
+mkdir -p bazel/bin
+mv bazelisk-linux-amd64 bazel/bin/bazel
+export BAZEL_VERSION=$(./bazel/bin/bazel --version | awk '{print $2}')
+export PATH=$PATH:$(pwd)/bazel/bin

.ci/env/environment.yml

@@ -0,0 +1,7 @@
+name: ci-env
+channels:
+ - conda-forge
+ - intel
+ - defaults
+dependencies:
+ - impi-devel=2021.10.0

.ci/pipeline/ci.yml

@@ -28,9 +28,7 @@ jobs:
  vmImage: 'ubuntu-22.04'
  steps:
  - script: |
- sudo apt-get update && sudo apt-get install -y clang-format-14
- sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-14 100
- sudo update-alternatives --set clang-format /usr/bin/clang-format-14
+ .ci/env/apt.sh clang-format
  displayName: 'apt-get'
  - script: |
  .ci/scripts/clang-format.sh
@@ -46,8 +44,7 @@ jobs:
  vmImage: 'ubuntu-22.04'
  steps:
  - script: |
- sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree
- conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.10.0 openjdk=17.0.3
+ .ci/env/apt.sh dev-base
  displayName: 'apt-get and conda install'
  - script: |
  .ci/scripts/describe_system.sh
@@ -95,8 +92,7 @@ jobs:
  vmImage: 'ubuntu-22.04'
  steps:
  - script: |
- sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree
- conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.8.0 openjdk=17.0.3
+ .ci/env/apt.sh dev-base
  displayName: 'apt-get and conda install'
  - script: |
  .ci/scripts/describe_system.sh
@@ -136,11 +132,10 @@ jobs:
  vmImage: 'ubuntu-22.04'
  steps:
  - script: |
- sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix
- conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 openjdk=17.0.3
+ .ci/env/apt.sh dev-base
  displayName: 'apt-get and conda install'
  - script: |
- bash .ci/scripts/install_dpcpp.sh
+ .ci/env/apt.sh dpcpp
  displayName: 'dpcpp installation'
  - script: |
  source /opt/intel/oneapi/compiler/latest/env/vars.sh
@@ -190,27 +185,21 @@ jobs:
  BAZEL_CACHE_MAX_SIZE_KB: 4194304 # Size in kilobytes ~ 4Gb
  steps:
  - script: |
- # Download Bazelisk
- export SHA256="168851e70cf5f95c0e215e7f3aaca5132ffc3c8dd8f585a4157b0be2b53cfe32 bazelisk-linux-amd64"
- wget https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64
- echo ${SHA256} | sha256sum --check
- # "Install" bazelisk
- chmod +x bazelisk-linux-amd64
- mv bazelisk-linux-amd64 ${BAZEL_VERSION}
-
- displayName: 'install-deps'
+ source .ci/env/bazelisk.sh
+ echo "##vso[task.setvariable variable=BAZEL_VERSION]${BAZEL_VERSION}"
+ #sourcing done to set bazel version value from script
+ displayName: 'install-bazel'
  - script: |
  .ci/scripts/describe_system.sh
  displayName: 'System info'
-
  - task: Cache@2
  inputs:
  # Commit ID is added to a cache key. Caches are immutable by design,
  # so we always need to change a key to upload the last version
  # of the Bazel cache. Cache lookup is based on `restoreKeys` option.
- key: '"$(BAZEL_VERSION)" | "$(Agent.OS)" | "v2" | "$(Build.SourceVersion)"'
+ key: '"$(BAZEL_VERSION)" | "$(Agent.OS)" | "v1" | "$(Build.SourceVersion)"'
  restoreKeys: |
- "$(BAZEL_VERSION)" | "$(Agent.OS)" | "v2"
+ "$(BAZEL_VERSION)" | "$(Agent.OS)" | "v1"
  path: $(BAZEL_CACHE_DIR)
  displayName: 'bazel-cache'
 
@@ -227,41 +216,41 @@ jobs:
  echo "build --disk_cache=$(BAZEL_CACHE_DIR) --cpu=avx2" > ~/.bazelrc
 
  # Display oneDAL build configuration
- $(BAZEL_VERSION) build @config//:dump
+ bazel build @config//:dump
  echo
  cat bazel-bin/external/config/config.json
  echo
  displayName: 'bazel-configure'
  - script: |
- $(BAZEL_VERSION) build :release
+ bazel build :release
  displayName: 'release'
  - task: PublishPipelineArtifact@1
  inputs:
  artifactName: '$(platform.type) Bazel build'
  targetPath: '$(Build.Repository.LocalPath)/bazel-bin/release'
  displayName: 'Upload build artifacts'
  - script: |
- $(BAZEL_VERSION) test //examples/oneapi/cpp:all \
+ bazel test //examples/oneapi/cpp:all \
  --test_link_mode=dev \
  --test_thread_mode=par
  displayName: 'cpp-examples-thread-dev'
 
  - script: |
  export DALROOT=`pwd`/bazel-bin/release/daal/latest
- $(BAZEL_VERSION) test //examples/oneapi/cpp:all \
+ bazel test //examples/oneapi/cpp:all \
  --test_link_mode=release_static \
  --test_thread_mode=par
  displayName: 'cpp-examples-thread-release-static'
 
  - script: |
  export DALROOT=`pwd`/bazel-bin/release/daal/latest
- $(BAZEL_VERSION) test //examples/oneapi/cpp:all \
+ bazel test //examples/oneapi/cpp:all \
  --test_link_mode=release_dynamic \
  --test_thread_mode=par
  displayName: 'cpp-examples-thread-release-dynamic'
 
  - script: |
- $(BAZEL_VERSION) test //cpp/oneapi/dal:tests \
+ bazel test //cpp/oneapi/dal:tests \
  --config=host \
  --test_link_mode=dev \
  --test_thread_mode=par
@@ -310,7 +299,7 @@ jobs:
  inputs:
  artifact: '$(platform.type) build'
  path: $(Pipeline.Workspace)
- - script: sudo apt-get update && sudo apt-get install -y git gcc-multilib g++-multilib
+ - script: .ci/env/apt.sh dev-base
  displayName: 'apt-get'
  - script: |
  .ci/scripts/describe_system.sh
@@ -392,7 +381,7 @@ jobs:
  steps:
  - script: |
  brew install dos2unix tree
- conda create -n ci-env -q -y -c conda-forge python=3.9 openjdk=17.0.3
+ conda create -n ci-env -q -y -c conda-forge python=3.9
  source /usr/local/miniconda/etc/profile.d/conda.sh
  conda activate ci-env
  pip install -q cpufeature

.github/workflows/docker-validation-ci.yml

@@ -0,0 +1,19 @@
+name: docker-validation CI
+
+on:
+ pull_request:
+ branches:
+ - master
+ push:
+ branches:
+ - master
+
+jobs:
+ validate:
+ name: Docker validation
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+ - name: Build docker image
+ run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest

.github/workflows/docker-validation-nightly.yml

@@ -0,0 +1,35 @@
+name: docker-validation Nightly
+
+on:
+ schedule:
+ - cron: "0 23 * * *"
+ pull_request:
+ branches:
+ - master
+ paths:
+ - .github/workflows/docker-validation-nightly.yml
+ - dev/docker/onedal-dev.Dockerfile
+ - .ci/env
+ - .ci/scripts
+ push:
+ branches:
+ - master
+ paths:
+ - .github/workflows/docker-validation-nightly.yml
+ - dev/docker/onedal-dev.Dockerfile
+ - .ci/env
+ - .ci/scripts
+
+jobs:
+ validate:
+ name: Docker validation
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+ - name: Build docker image
+ run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest
+ - name: Building oneDAL
+ run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env
+ - name: Building oneDAL with bazel
+ run: docker run onedal-dev bazel build :release

INSTALL.md

@@ -28,6 +28,11 @@ Required Software:
 
 For details, see [System Requirements for oneDAL](https://www.intel.com/content/www/us/en/developer/articles/system-requirements/system-requirements-for-oneapi-data-analytics-library.html).
 
+## Docker Development Environment 
+
+[Docker file](https://github.com/oneapi-src/oneDAL/tree/master/dev) with the oneDAL development environment
+is available as an alternative to the manual setup. 
+
 ## Installation Steps
 1. Clone the sources from GitHub\* as follows:
 

dev/bazel/README.md

@@ -1,17 +1,33 @@
+<!--
+******************************************************************************
+* Copyright 2023 Intel Corporation
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*******************************************************************************/-->
 # Bazel Guide
 ## Install Bazel on Linux
-1. Download Bazel 4.0.0
+1. Use Bazelisk
  ```sh
- wget -O bazel https://github.com/bazelbuild/bazel/releases/download/4.0.0/bazel-4.0.0-linux-x86_64
+ wget -O bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64
  ```
  > Note: If you are using proxy don't forget to set
  `http_proxy` and `https_proxy` environment variables
 
 2. Put it somewhere on **local disk** (use of remote shared
  drives is not recommended) and set executable attributes.
  ```sh
- mv bazel <my-user-dir-utils>/bin
  chmod +x bazel
+ mv bazel <my-user-dir-utils>/bin
  ```
 
 3. Create Bazel cache directory on **local disk** and
@@ -24,7 +40,7 @@
 4. Add `bazel` to the `$PATH`.
  ```sh
  export PATH=<my-user-dir-on-local-disk>/bin:$PATH
- bazel --version # Should be "bazel 4.0.0"
+ bazel --version
  ```
 
 ### Compiler choice
@@ -197,10 +213,6 @@ The most used Bazel commands are `build`, `test` and `run`.
  bazel test --test_link_mode=release_dynamic //cpp/oneapi/dal:tests
  ```
 
-- `--test_thread_mode` Specifies threading mode for tests. \
- Possible values:
- - `par` _default_ Links against `onedal_thread`.
-
  Example:
  ```sh
  bazel test --test_thread_mode=par //cpp/oneapi/dal:tests