From ce16228a6e5e0eb2f96a43c51d99018366363763 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 13:16:35 +0100 Subject: [PATCH 01/48] removing JDK deps --- .ci/pipeline/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 5ccf75f76b0..0a3351629d9 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -47,7 +47,7 @@ jobs: steps: - script: | sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree - conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.10.0 openjdk=17.0.3 + conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.10.0 displayName: 'apt-get and conda install' - script: | .ci/scripts/describe_system.sh @@ -96,7 +96,7 @@ jobs: steps: - script: | sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree - conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.8.0 openjdk=17.0.3 + conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.8.0 displayName: 'apt-get and conda install' - script: | .ci/scripts/describe_system.sh @@ -137,7 +137,7 @@ jobs: steps: - script: | sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix - conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 openjdk=17.0.3 + conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 displayName: 'apt-get and conda install' - script: | bash .ci/scripts/install_dpcpp.sh @@ -392,7 +392,7 @@ jobs: steps: - script: | brew install dos2unix tree - conda create -n ci-env -q -y -c conda-forge python=3.9 openjdk=17.0.3 + conda create -n ci-env -q -y -c conda-forge python=3.9 source /usr/local/miniconda/etc/profile.d/conda.sh conda activate ci-env pip install -q cpufeature From b02a52f6aee3b8a3b875a4228e4445fdcf2d0fcf Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 13:35:58 +0100 Subject: [PATCH 02/48] Fixing env and introduce docker --- .ci/env/{oneapi.sh => apt.sh} | 31 +++++++++++++------ .ci/pipeline/ci.yml | 14 ++++----- .../docker/onedal.Dockerfile | 21 ++++++------- 3 files changed, 38 insertions(+), 28 deletions(-) rename .ci/env/{oneapi.sh => apt.sh} (71%) mode change 100755 => 100644 rename .ci/scripts/install_dpcpp.sh => dev/docker/onedal.Dockerfile (51%) mode change 100755 => 100644 diff --git a/.ci/env/oneapi.sh b/.ci/env/apt.sh old mode 100755 new mode 100644 similarity index 71% rename from .ci/env/oneapi.sh rename to .ci/env/apt.sh index 55ecdc9f3ed..45d56508676 --- a/.ci/env/oneapi.sh +++ b/.ci/env/apt.sh @@ -17,6 +17,10 @@ component=$1 +function update { + sudo apt-get update +} + function add_repo { wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB @@ -27,14 +31,7 @@ function add_repo { } function install_dpcpp { - sudo apt-get install \ - intel-oneapi-common-vars \ - intel-oneapi-common-licensing \ - intel-oneapi-tbb-devel \ - intel-oneapi-dpcpp-cpp-compiler \ - intel-oneapi-dev-utilities \ - intel-oneapi-libdpstd-devel \ - cmake + sudo apt-get install -y intel-dpcpp-cpp-compiler-2023.2.1 sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd' sudo mv -f /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga_ } @@ -43,14 +40,30 @@ function install_mkl { sudo apt-get install intel-oneapi-mkl-devel } +function install_clang-format { + sudo apt-get install -y clang-format-14 + sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-14 100 + sudo update-alternatives --set clang-format /usr/bin/clang-format-14 +} + +function install_dev-base { + sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree +} + if [ "${component}" == "dpcpp" ]; then add_repo install_dpcpp elif [ "${component}" == "mkl" ]; then add_repo install_mkl +elif [ "${component}" == "clang-format" ]; then + update + install_clang-format +elif [ "${component}" == "dev-base" ]; then + update + install_dev-base else echo "Usage:" - echo " $0 [dpcpp|mkl]" + echo " $0 [dpcpp|mkl|clang-format|dev-base]" exit 1 fi diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 0a3351629d9..75236225e68 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -28,9 +28,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - sudo apt-get update && sudo apt-get install -y clang-format-14 - sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-14 100 - sudo update-alternatives --set clang-format /usr/bin/clang-format-14 + bash .ci/env/apt.sh clang-format displayName: 'apt-get' - script: | .ci/scripts/clang-format.sh @@ -46,7 +44,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree + bash .ci/env/apt.sh dev-base conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.10.0 displayName: 'apt-get and conda install' - script: | @@ -95,7 +93,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree + bash .ci/env/apt.sh dev-base conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.8.0 displayName: 'apt-get and conda install' - script: | @@ -136,11 +134,11 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix + bash .ci/env/apt.sh dev-base conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 displayName: 'apt-get and conda install' - script: | - bash .ci/scripts/install_dpcpp.sh + bash .ci/env/apt.sh install_dpcpp displayName: 'dpcpp installation' - script: | source /opt/intel/oneapi/compiler/latest/env/vars.sh @@ -310,7 +308,7 @@ jobs: inputs: artifact: '$(platform.type) build' path: $(Pipeline.Workspace) - - script: sudo apt-get update && sudo apt-get install -y git gcc-multilib g++-multilib + - script: bash .ci/env/apt.sh dev-base displayName: 'apt-get' - script: | .ci/scripts/describe_system.sh diff --git a/.ci/scripts/install_dpcpp.sh b/dev/docker/onedal.Dockerfile old mode 100755 new mode 100644 similarity index 51% rename from .ci/scripts/install_dpcpp.sh rename to dev/docker/onedal.Dockerfile index 7df36449d9d..e36e0faf636 --- a/.ci/scripts/install_dpcpp.sh +++ b/dev/docker/onedal.Dockerfile @@ -1,6 +1,5 @@ -#!/bin/bash #=============================================================================== -# Copyright 2023 Intel Corporation +# Copyright 2014 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,12 +14,12 @@ # limitations under the License. #=============================================================================== -wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB -sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB -rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB -echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list -sudo add-apt-repository -y "deb https://apt.repos.intel.com/oneapi all main" -sudo apt-get update -sudo apt-get install -y intel-dpcpp-cpp-compiler-2023.2.1 -sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd' -sudo mv -f /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga_ +ARG UBUNTU_VERSION="22.04" + +FROM ubuntu:${UBUNTU_VERSION} + +RUN apt-get update && \ + apt-get install -y \ + git + +RUN git checkout \ No newline at end of file From f986493da58006ed2cf0f423ac6ffc35bab66502 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:21:24 +0100 Subject: [PATCH 03/48] Creating bazel env script --- .ci/env/bazelisk.sh | 27 +++++++++++++++++++ .ci/pipeline/ci.yml | 13 +++------ ...nedal.Dockerfile => onedal-dev.Dockerfile} | 22 ++++++++++----- 3 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 .ci/env/bazelisk.sh rename dev/docker/{onedal.Dockerfile => onedal-dev.Dockerfile} (63%) diff --git a/.ci/env/bazelisk.sh b/.ci/env/bazelisk.sh new file mode 100644 index 00000000000..9c95af24070 --- /dev/null +++ b/.ci/env/bazelisk.sh @@ -0,0 +1,27 @@ +#!/bin/bash +#=============================================================================== +# Copyright 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + +# Download Bazelisk +export SHA256="ce52caa51ef9e509fb6b7e5ad892e5cf10feb0794b0aed4d2f36adb00a1a2779 bazelisk-linux-amd64" +wget https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 +echo ${SHA256} | sha256sum --check +# "Install" bazelisk +chmod +x bazelisk-linux-amd64 +mkdir -p bazel/bin +mv bazelisk-linux-amd64 bazel/bin/bazel +export BAZEL_VERSION=$(./bazel/bin/bazel --version | awk '{print $2}') +export PATH=$PATH:$(pwd)/bazel/bin \ No newline at end of file diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 75236225e68..516e8d6eddf 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -188,19 +188,12 @@ jobs: BAZEL_CACHE_MAX_SIZE_KB: 4194304 # Size in kilobytes ~ 4Gb steps: - script: | - # Download Bazelisk - export SHA256="168851e70cf5f95c0e215e7f3aaca5132ffc3c8dd8f585a4157b0be2b53cfe32 bazelisk-linux-amd64" - wget https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 - echo ${SHA256} | sha256sum --check - # "Install" bazelisk - chmod +x bazelisk-linux-amd64 - mv bazelisk-linux-amd64 ${BAZEL_VERSION} - - displayName: 'install-deps' + .ci/env/bazelisk.sh + echo "##vso[task.setvariable variable=BAZEL_VERSION]${BAZEL_VERSION} + displayName: 'install-bazel' - script: | .ci/scripts/describe_system.sh displayName: 'System info' - - task: Cache@2 inputs: # Commit ID is added to a cache key. Caches are immutable by design, diff --git a/dev/docker/onedal.Dockerfile b/dev/docker/onedal-dev.Dockerfile similarity index 63% rename from dev/docker/onedal.Dockerfile rename to dev/docker/onedal-dev.Dockerfile index e36e0faf636..2bafc11d6dc 100644 --- a/dev/docker/onedal.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -14,12 +14,22 @@ # limitations under the License. #=============================================================================== -ARG UBUNTU_VERSION="22.04" +FROM ubuntu:22.04 -FROM ubuntu:${UBUNTU_VERSION} +ARG workdirectory="/sources/oneDAL" -RUN apt-get update && \ - apt-get install -y \ - git +ADD ../../ ${workdirectory} -RUN git checkout \ No newline at end of file +WORKDIR ${workdirectory} + +# Installing environment for base development dependencies +RUN .ci/env/apt.sh dev-base + +# Installing environment for DPCPP development dependencies +RUN .ci/env/apt.sh install_dpcpp + +# Installing environment for clang-format +RUN .ci/env/apt.sh clang-format + +# Installing environment for bazel +RUN .ci/env/bazilisk.sh From 5146f7b9f64a0f0d53657a18e7eae6bd48d97c48 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:25:12 +0100 Subject: [PATCH 04/48] adding readme for docker --- dev/docker/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 dev/docker/README.md diff --git a/dev/docker/README.md b/dev/docker/README.md new file mode 100644 index 00000000000..e0092d9ad21 --- /dev/null +++ b/dev/docker/README.md @@ -0,0 +1,26 @@ + + +# Docker dev environemnt +## How to use +This is simple docker dev environment intended for oneDAL development and build process. +It does include dependencies for building all oneDAL components through both make and bazel + +Just run: + ```sh + docker run onedal-dev + ``` From 23ff0001c2203223e19fea963dddb8e07a88650c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:37:12 +0100 Subject: [PATCH 05/48] updating --- dev/docker/README.md | 4 +++- dev/docker/onedal-dev.Dockerfile | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/docker/README.md b/dev/docker/README.md index e0092d9ad21..9fa9472eedb 100644 --- a/dev/docker/README.md +++ b/dev/docker/README.md @@ -20,7 +20,9 @@ This is simple docker dev environment intended for oneDAL development and build process. It does include dependencies for building all oneDAL components through both make and bazel +Note: docker setup assumes that it's executed from oneDAL checkouted repo and copy repo files inside container + Just run: ```sh - docker run onedal-dev + docker run -it onedal-dev /bin/bash ``` diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 2bafc11d6dc..8e46d228940 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -22,6 +22,8 @@ ADD ../../ ${workdirectory} WORKDIR ${workdirectory} +#Env setup + # Installing environment for base development dependencies RUN .ci/env/apt.sh dev-base From cf15bcec3107c03cc69cc6142f9aac637d174f9b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:41:44 +0100 Subject: [PATCH 06/48] adding github validation --- .github/workflows/docker-validation.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/docker-validation.yml diff --git a/.github/workflows/docker-validation.yml b/.github/workflows/docker-validation.yml new file mode 100644 index 00000000000..ab6fce8ac58 --- /dev/null +++ b/.github/workflows/docker-validation.yml @@ -0,0 +1,19 @@ +name: renovate-validation + +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + validate: + name: Docker dev validation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + - name: Build docker image + run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:$(date +%s) From 39ae618b6bc80b9444b802447af040d8688b6286 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:48:19 +0100 Subject: [PATCH 07/48] new line --- .ci/env/bazelisk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/env/bazelisk.sh b/.ci/env/bazelisk.sh index 9c95af24070..0d772327b75 100644 --- a/.ci/env/bazelisk.sh +++ b/.ci/env/bazelisk.sh @@ -24,4 +24,4 @@ chmod +x bazelisk-linux-amd64 mkdir -p bazel/bin mv bazelisk-linux-amd64 bazel/bin/bazel export BAZEL_VERSION=$(./bazel/bin/bazel --version | awk '{print $2}') -export PATH=$PATH:$(pwd)/bazel/bin \ No newline at end of file +export PATH=$PATH:$(pwd)/bazel/bin From 74fefc7a75138845906072bcf88d7488fdb195d9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:49:43 +0100 Subject: [PATCH 08/48] remove bash invocation for scripts --- .ci/pipeline/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 516e8d6eddf..edf1ed5ccff 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -28,7 +28,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - bash .ci/env/apt.sh clang-format + .ci/env/apt.sh clang-format displayName: 'apt-get' - script: | .ci/scripts/clang-format.sh @@ -44,7 +44,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - bash .ci/env/apt.sh dev-base + .ci/env/apt.sh dev-base conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.10.0 displayName: 'apt-get and conda install' - script: | @@ -93,7 +93,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - bash .ci/env/apt.sh dev-base + .ci/env/apt.sh dev-base conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.8.0 displayName: 'apt-get and conda install' - script: | @@ -134,11 +134,11 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - bash .ci/env/apt.sh dev-base + .ci/env/apt.sh dev-base conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 displayName: 'apt-get and conda install' - script: | - bash .ci/env/apt.sh install_dpcpp + .ci/env/apt.sh install_dpcpp displayName: 'dpcpp installation' - script: | source /opt/intel/oneapi/compiler/latest/env/vars.sh @@ -301,7 +301,7 @@ jobs: inputs: artifact: '$(platform.type) build' path: $(Pipeline.Workspace) - - script: bash .ci/env/apt.sh dev-base + - script: .ci/env/apt.sh dev-base displayName: 'apt-get' - script: | .ci/scripts/describe_system.sh @@ -439,7 +439,7 @@ jobs: displayName: 'pacman' - script: | set PATH=C:\msys64\usr\bin;%PATH% - bash .ci/scripts/describe_system.sh + .ci/scripts/describe_system.sh displayName: 'System info' - script: | .\.ci\scripts\build.bat daal vc avx2 @@ -463,7 +463,7 @@ jobs: displayName: 'oneapi/cpp examples' - script: | set PATH=C:\msys64\usr\bin;%PATH% - bash deploy/nuget/prepare_dal_nuget.sh --release-dir $(release.dir) --build-nupkg yes + deploy/nuget/prepare_dal_nuget.sh --release-dir $(release.dir) --build-nupkg yes tree -h -I include __nuget/inteldal*/ ls -lh __nuget/inteldal*.nupkg displayName: 'nuget pkg' From 94986bc06e66ae582104d1094414e57c6741c72f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:50:59 +0100 Subject: [PATCH 09/48] fixing ci.yml --- .ci/pipeline/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index edf1ed5ccff..6f6693cf015 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -28,7 +28,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - .ci/env/apt.sh clang-format + .ci/env/apt.sh clang-format displayName: 'apt-get' - script: | .ci/scripts/clang-format.sh From 21ea80e2fd1ffdb0c213814b42f833e0642a2275 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:53:19 +0100 Subject: [PATCH 10/48] add execute permissions --- .ci/env/apt.sh | 0 .ci/env/bazelisk.sh | 0 .ci/env/tbb.bat | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .ci/env/apt.sh mode change 100644 => 100755 .ci/env/bazelisk.sh mode change 100644 => 100755 .ci/env/tbb.bat diff --git a/.ci/env/apt.sh b/.ci/env/apt.sh old mode 100644 new mode 100755 diff --git a/.ci/env/bazelisk.sh b/.ci/env/bazelisk.sh old mode 100644 new mode 100755 diff --git a/.ci/env/tbb.bat b/.ci/env/tbb.bat old mode 100644 new mode 100755 From e26db8441d5735a98a3000640fd80ca7254eb568 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:54:46 +0100 Subject: [PATCH 11/48] fixing ci.yml --- .ci/pipeline/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 6f6693cf015..133d74b1036 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -188,8 +188,8 @@ jobs: BAZEL_CACHE_MAX_SIZE_KB: 4194304 # Size in kilobytes ~ 4Gb steps: - script: | - .ci/env/bazelisk.sh - echo "##vso[task.setvariable variable=BAZEL_VERSION]${BAZEL_VERSION} + .ci/env/bazelisk.sh + echo "##vso[task.setvariable variable=BAZEL_VERSION]${BAZEL_VERSION} displayName: 'install-bazel' - script: | .ci/scripts/describe_system.sh From 5a0b07ffe6a81c7636933fec9659b1815c56d54d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:56:12 +0100 Subject: [PATCH 12/48] fixing github flow name --- .github/workflows/docker-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-validation.yml b/.github/workflows/docker-validation.yml index ab6fce8ac58..b99ecdb7473 100644 --- a/.github/workflows/docker-validation.yml +++ b/.github/workflows/docker-validation.yml @@ -1,4 +1,4 @@ -name: renovate-validation +name: docker-validation on: pull_request: From 35ea7b885753b3a75325bff19f2790c0ae2997ad Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 17:57:46 +0100 Subject: [PATCH 13/48] adding sudo to docker image --- dev/docker/onedal-dev.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 8e46d228940..ea02c289738 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -23,6 +23,8 @@ ADD ../../ ${workdirectory} WORKDIR ${workdirectory} #Env setup +RUN apt-get update && \ + apt-get -y install sudo # Installing environment for base development dependencies RUN .ci/env/apt.sh dev-base From 1d6a5af8c446dc459619992fb4cb2b456efc4223 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 18:00:47 +0100 Subject: [PATCH 14/48] fixing install dpcpp script --- .ci/pipeline/ci.yml | 2 +- dev/docker/onedal-dev.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 133d74b1036..4f1d878ed88 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -138,7 +138,7 @@ jobs: conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 displayName: 'apt-get and conda install' - script: | - .ci/env/apt.sh install_dpcpp + .ci/env/apt.sh dpcpp displayName: 'dpcpp installation' - script: | source /opt/intel/oneapi/compiler/latest/env/vars.sh diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index ea02c289738..432585cb7b5 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && \ RUN .ci/env/apt.sh dev-base # Installing environment for DPCPP development dependencies -RUN .ci/env/apt.sh install_dpcpp +RUN .ci/env/apt.sh dpcpp # Installing environment for clang-format RUN .ci/env/apt.sh clang-format From c4f38e31071d72672667d761acb4db343d9685e0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 18:04:41 +0100 Subject: [PATCH 15/48] fixing base docker deps --- dev/docker/onedal-dev.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 432585cb7b5..529d7fae4e3 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -24,7 +24,7 @@ WORKDIR ${workdirectory} #Env setup RUN apt-get update && \ - apt-get -y install sudo + apt-get -y install sudo wget gnupg # Installing environment for base development dependencies RUN .ci/env/apt.sh dev-base From 4e8ffe88f9bb970e411918eec79b8729943da7e2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 18:15:34 +0100 Subject: [PATCH 16/48] fixing bazelisk --- dev/docker/onedal-dev.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 529d7fae4e3..78d48d7ce92 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -36,4 +36,4 @@ RUN .ci/env/apt.sh dpcpp RUN .ci/env/apt.sh clang-format # Installing environment for bazel -RUN .ci/env/bazilisk.sh +RUN .ci/env/bazelisk.sh From dbfb9fb5bb22834ef19e8a071e0a10456e7a83f3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 18:38:04 +0100 Subject: [PATCH 17/48] fixing ci.yml --- .ci/pipeline/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 4f1d878ed88..9db05a061e4 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -189,7 +189,7 @@ jobs: steps: - script: | .ci/env/bazelisk.sh - echo "##vso[task.setvariable variable=BAZEL_VERSION]${BAZEL_VERSION} + echo "##vso[task.setvariable variable=BAZEL_VERSION]${BAZEL_VERSION}" displayName: 'install-bazel' - script: | .ci/scripts/describe_system.sh From 08f75a35599e912ed0abafb56bdedec1a6f19c6b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 18:50:27 +0100 Subject: [PATCH 18/48] fixing windows ci with bash call --- .ci/pipeline/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 9db05a061e4..c08e8ac587e 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -439,7 +439,7 @@ jobs: displayName: 'pacman' - script: | set PATH=C:\msys64\usr\bin;%PATH% - .ci/scripts/describe_system.sh + bash .ci/scripts/describe_system.sh displayName: 'System info' - script: | .\.ci\scripts\build.bat daal vc avx2 From bec07464dfe153b3fb8028b30f1ec4fea4c88b38 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 18:53:40 +0100 Subject: [PATCH 19/48] fixing bazel ci --- .ci/pipeline/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index c08e8ac587e..0cf22f8d1ab 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -218,13 +218,13 @@ jobs: echo "build --disk_cache=$(BAZEL_CACHE_DIR) --cpu=avx2" > ~/.bazelrc # Display oneDAL build configuration - $(BAZEL_VERSION) build @config//:dump + bazel build @config//:dump echo cat bazel-bin/external/config/config.json echo displayName: 'bazel-configure' - script: | - $(BAZEL_VERSION) build :release + bazel build :release displayName: 'release' - task: PublishPipelineArtifact@1 inputs: @@ -232,27 +232,27 @@ jobs: targetPath: '$(Build.Repository.LocalPath)/bazel-bin/release' displayName: 'Upload build artifacts' - script: | - $(BAZEL_VERSION) test //examples/oneapi/cpp:all \ + bazel test //examples/oneapi/cpp:all \ --test_link_mode=dev \ --test_thread_mode=par displayName: 'cpp-examples-thread-dev' - script: | export DALROOT=`pwd`/bazel-bin/release/daal/latest - $(BAZEL_VERSION) test //examples/oneapi/cpp:all \ + bazel test //examples/oneapi/cpp:all \ --test_link_mode=release_static \ --test_thread_mode=par displayName: 'cpp-examples-thread-release-static' - script: | export DALROOT=`pwd`/bazel-bin/release/daal/latest - $(BAZEL_VERSION) test //examples/oneapi/cpp:all \ + bazel test //examples/oneapi/cpp:all \ --test_link_mode=release_dynamic \ --test_thread_mode=par displayName: 'cpp-examples-thread-release-dynamic' - script: | - $(BAZEL_VERSION) test //cpp/oneapi/dal:tests \ + bazel test //cpp/oneapi/dal:tests \ --config=host \ --test_link_mode=dev \ --test_thread_mode=par From c032de6cca413974bf610ca9bc6951a28356f437 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 19:13:05 +0100 Subject: [PATCH 20/48] updating bazel readme to current state --- dev/bazel/README.md | 28 ++++++++++++++++++++-------- dev/bazel/TODO.md | 26 ++++++++++++++++++-------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/dev/bazel/README.md b/dev/bazel/README.md index e806f0d39e8..2593342d3e6 100644 --- a/dev/bazel/README.md +++ b/dev/bazel/README.md @@ -1,8 +1,24 @@ + # Bazel Guide ## Install Bazel on Linux -1. Download Bazel 4.0.0 +1. Use bazelisk ```sh - wget -O bazel https://github.com/bazelbuild/bazel/releases/download/4.0.0/bazel-4.0.0-linux-x86_64 + wget -O bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 ``` > Note: If you are using proxy don't forget to set `http_proxy` and `https_proxy` environment variables @@ -10,8 +26,8 @@ 2. Put it somewhere on **local disk** (use of remote shared drives is not recommended) and set executable attributes. ```sh - mv bazel /bin chmod +x bazel + mv bazel /bin ``` 3. Create Bazel cache directory on **local disk** and @@ -24,7 +40,7 @@ 4. Add `bazel` to the `$PATH`. ```sh export PATH=/bin:$PATH - bazel --version # Should be "bazel 4.0.0" + bazel --version ``` ### Compiler choice @@ -197,10 +213,6 @@ The most used Bazel commands are `build`, `test` and `run`. bazel test --test_link_mode=release_dynamic //cpp/oneapi/dal:tests ``` -- `--test_thread_mode` Specifies threading mode for tests. \ - Possible values: - - `par` _default_ Links against `onedal_thread`. - Example: ```sh bazel test --test_thread_mode=par //cpp/oneapi/dal:tests diff --git a/dev/bazel/TODO.md b/dev/bazel/TODO.md index 34a71a5d0bb..4a81faf5a5b 100644 --- a/dev/bazel/TODO.md +++ b/dev/bazel/TODO.md @@ -1,3 +1,20 @@ + + ### TODO: - [ ] **Windows support.** Bazel shall provide toolchain implementation for Windows. @@ -9,18 +26,11 @@ | | Intel | DPC++ | GCC | Clang | MSVC | |---------|:------------------:|:------------------:|:------------------:|:------------------:|:------------------:| | Linux | :x: | :x: | :heavy_check_mark: | | | - | MacOs | :x: | :x: | | :x: | | | Windows | :x: | :x: | | | :x: | -- [ ] **Automatic makefile generation for examples on Linux/MacOs.** Bazel - shall generate Makefile for examples when release build is required. - -- [ ] **Automatic VS solution generation for examples on Windows.** Bazel shall - generate Visual Studio solution for examples then release build is required. - - [ ] **Automatic host architecture identification.** Bazel shall detect host machine architecture and configure best CPU id automatically. - [ ] **Toolchain code unification.** There is logic duplication for toolchain - configuration on Linux/MacOs/Windows. + configuration on Linux/Windows. From aaa1ace79326609b840626b8d0657efe0fe701aa Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 13 Sep 2023 20:01:30 +0100 Subject: [PATCH 21/48] fixing bazel caching and nuget windows build --- .ci/pipeline/ci.yml | 5 +++-- dev/docker/onedal-dev.Dockerfile | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 0cf22f8d1ab..a0f211bf071 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -188,8 +188,9 @@ jobs: BAZEL_CACHE_MAX_SIZE_KB: 4194304 # Size in kilobytes ~ 4Gb steps: - script: | - .ci/env/bazelisk.sh + source .ci/env/bazelisk.sh echo "##vso[task.setvariable variable=BAZEL_VERSION]${BAZEL_VERSION}" + #sourcing done to set bazel version value from script displayName: 'install-bazel' - script: | .ci/scripts/describe_system.sh @@ -463,7 +464,7 @@ jobs: displayName: 'oneapi/cpp examples' - script: | set PATH=C:\msys64\usr\bin;%PATH% - deploy/nuget/prepare_dal_nuget.sh --release-dir $(release.dir) --build-nupkg yes + bash deploy/nuget/prepare_dal_nuget.sh --release-dir $(release.dir) --build-nupkg yes tree -h -I include __nuget/inteldal*/ ls -lh __nuget/inteldal*.nupkg displayName: 'nuget pkg' diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 78d48d7ce92..939ef0f6765 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -37,3 +37,6 @@ RUN .ci/env/apt.sh clang-format # Installing environment for bazel RUN .ci/env/bazelisk.sh + +# Installing openBLAS dependency +RUN .ci/env/openblas.sh From f828fada56fb96dd4c4a3a51d4402a566c163810 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 14 Sep 2023 10:38:40 +0100 Subject: [PATCH 22/48] adding git binary --- dev/docker/onedal-dev.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 939ef0f6765..2e830fdbfa2 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -24,7 +24,7 @@ WORKDIR ${workdirectory} #Env setup RUN apt-get update && \ - apt-get -y install sudo wget gnupg + apt-get -y install sudo wget gnupg git # Installing environment for base development dependencies RUN .ci/env/apt.sh dev-base From 87a55e3d8fe7f295fe175e63966a3566927992d5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 14 Sep 2023 11:19:43 +0100 Subject: [PATCH 23/48] adding build validation and conda to env --- .ci/env/apt.sh | 5 +++++ .ci/pipeline/ci.yml | 3 --- .github/workflows/docker-validation.yml | 19 ------------------- dev/docker/onedal-dev.Dockerfile | 12 +++++++++++- 4 files changed, 16 insertions(+), 23 deletions(-) delete mode 100644 .github/workflows/docker-validation.yml diff --git a/.ci/env/apt.sh b/.ci/env/apt.sh index 45d56508676..3e7230ad9ca 100755 --- a/.ci/env/apt.sh +++ b/.ci/env/apt.sh @@ -50,6 +50,10 @@ function install_dev-base { sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree } +function install_dev-base-conda { + conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 +} + if [ "${component}" == "dpcpp" ]; then add_repo install_dpcpp @@ -62,6 +66,7 @@ elif [ "${component}" == "clang-format" ]; then elif [ "${component}" == "dev-base" ]; then update install_dev-base + install_dev-base-conda else echo "Usage:" echo " $0 [dpcpp|mkl|clang-format|dev-base]" diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index a0f211bf071..9d0a5456194 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -45,7 +45,6 @@ jobs: steps: - script: | .ci/env/apt.sh dev-base - conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.10.0 displayName: 'apt-get and conda install' - script: | .ci/scripts/describe_system.sh @@ -94,7 +93,6 @@ jobs: steps: - script: | .ci/env/apt.sh dev-base - conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.8.0 displayName: 'apt-get and conda install' - script: | .ci/scripts/describe_system.sh @@ -135,7 +133,6 @@ jobs: steps: - script: | .ci/env/apt.sh dev-base - conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 displayName: 'apt-get and conda install' - script: | .ci/env/apt.sh dpcpp diff --git a/.github/workflows/docker-validation.yml b/.github/workflows/docker-validation.yml deleted file mode 100644 index b99ecdb7473..00000000000 --- a/.github/workflows/docker-validation.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: docker-validation - -on: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - validate: - name: Docker dev validation - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 - - name: Build docker image - run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:$(date +%s) diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 2e830fdbfa2..92b89105cff 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -1,5 +1,5 @@ #=============================================================================== -# Copyright 2014 Intel Corporation +# Copyright 2023 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,3 +40,13 @@ RUN .ci/env/bazelisk.sh # Installing openBLAS dependency RUN .ci/env/openblas.sh + +ARG config +RUN \ +if [[ "${config}" == "build_make" ]] ; \ +then \ + .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env; \ + .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c; \ +else \ + echo Welcome to oneDAL development, sources located in ${workdirectory} ; \ +fi From 4765f766fc9c1033ad6d901516494efe5196d411 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 14 Sep 2023 11:23:44 +0100 Subject: [PATCH 24/48] enable larger validation in CI --- .github/workflows/docker-validation-ci.yml | 19 +++++++++++++ .../workflows/docker-validation-nightly.yml | 27 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .github/workflows/docker-validation-ci.yml create mode 100644 .github/workflows/docker-validation-nightly.yml diff --git a/.github/workflows/docker-validation-ci.yml b/.github/workflows/docker-validation-ci.yml new file mode 100644 index 00000000000..e8f76a63797 --- /dev/null +++ b/.github/workflows/docker-validation-ci.yml @@ -0,0 +1,19 @@ +name: docker-validation CI + +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + validate: + name: Docker validation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + - name: Build docker image + run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:$(date +%s) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml new file mode 100644 index 00000000000..7ff083a99c9 --- /dev/null +++ b/.github/workflows/docker-validation-nightly.yml @@ -0,0 +1,27 @@ +name: docker-validation Nightly + +on: + schedule: + - cron: "0 23 * * *" + pull_request: + branches: + - master + paths: + - .github/workflows/docker-validation-nightly.yml + - dev/docker/onedal-dev.Dockerfile + push: + branches: + - master + paths: + - .github/workflows/docker-validation-nightly.yml + - dev/docker/onedal-dev.Dockerfile + +jobs: + validate: + name: Docker validation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + - name: Build docker image + run: docker build . --file dev/docker/onedal-dev.Dockerfile -build-arg config=build_make --tag onedal-dev:$(date +%s) From 8a395f856b71b38e84b58083230d74611fc9f643 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 14 Sep 2023 11:25:23 +0100 Subject: [PATCH 25/48] fixing CLI --- .github/workflows/docker-validation-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index 7ff083a99c9..00e5e2baba4 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -24,4 +24,4 @@ jobs: - name: Checkout uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 - name: Build docker image - run: docker build . --file dev/docker/onedal-dev.Dockerfile -build-arg config=build_make --tag onedal-dev:$(date +%s) + run: docker build . --file dev/docker/onedal-dev.Dockerfile --build-arg config=build_make --tag onedal-dev:$(date +%s) From f726d9993ca101a81538953e90fb28aa3f487743 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 14 Sep 2023 11:31:17 +0100 Subject: [PATCH 26/48] adding conda to docker --- dev/docker/onedal-dev.Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 92b89105cff..0841b8847de 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -26,6 +26,14 @@ WORKDIR ${workdirectory} RUN apt-get update && \ apt-get -y install sudo wget gnupg git +# Install miniconda +ENV CONDA_DIR /opt/conda +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda + +# Put conda in path so we can use conda activate +ENV PATH=$CONDA_DIR/bin:$PATH + # Installing environment for base development dependencies RUN .ci/env/apt.sh dev-base From da719efebe8cf308b89447e022be39484a04e789 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 14 Sep 2023 15:47:28 +0100 Subject: [PATCH 27/48] fixing docker --- dev/docker/onedal-dev.Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 0841b8847de..dfd881e8691 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -24,7 +24,7 @@ WORKDIR ${workdirectory} #Env setup RUN apt-get update && \ - apt-get -y install sudo wget gnupg git + apt-get -y install sudo wget gnupg git make # Install miniconda ENV CONDA_DIR /opt/conda @@ -50,8 +50,9 @@ RUN .ci/env/bazelisk.sh RUN .ci/env/openblas.sh ARG config + RUN \ -if [[ "${config}" == "build_make" ]] ; \ +if [ "${config}" == "build_make" ] ; \ then \ .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env; \ .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c; \ From 6f7d25e0970afcc380cb11a89aa709ab3c66cb7e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 14 Sep 2023 18:28:01 +0100 Subject: [PATCH 28/48] fixing docker --- dev/docker/onedal-dev.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index dfd881e8691..60ae1ab2a10 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -52,7 +52,7 @@ RUN .ci/env/openblas.sh ARG config RUN \ -if [ "${config}" == "build_make" ] ; \ +if [ "${config}" = "build_make" ] ; \ then \ .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env; \ .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c; \ From 9a4049eca50450af0c64a282bb334245003335a9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 12:03:45 +0100 Subject: [PATCH 29/48] adding deps and build --- .github/workflows/docker-validation-nightly.yml | 2 ++ INSTALL.md | 5 +++++ dev/docker/onedal-dev.Dockerfile | 14 +++++++------- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index 00e5e2baba4..328a000d9fd 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -25,3 +25,5 @@ jobs: uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 - name: Build docker image run: docker build . --file dev/docker/onedal-dev.Dockerfile --build-arg config=build_make --tag onedal-dev:$(date +%s) + - name: Save docker image + run: docker save --output onedal-dev.tar onedal-dev diff --git a/INSTALL.md b/INSTALL.md index b42ae9f988c..fef8bd4fca0 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -28,6 +28,11 @@ Required Software: For details, see [System Requirements for oneDAL](https://www.intel.com/content/www/us/en/developer/articles/system-requirements/system-requirements-for-oneapi-data-analytics-library.html). +## Docker development environement + +[Docker file](https://github.com/oneapi-src/oneDAL/tree/master/dev) with oneDAL development environemnt +is avaialbe as alternative for setting up environemnt manually. + ## Installation Steps 1. Clone the sources from GitHub\* as follows: diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 60ae1ab2a10..c3edfe30517 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -49,13 +49,13 @@ RUN .ci/env/bazelisk.sh # Installing openBLAS dependency RUN .ci/env/openblas.sh -ARG config +# Installing MKL dependency +RUN ./dev/download_micromkl.sh +# Installing TBB dependency +RUN ./dev/download_tbb.sh + +# Temporary testing RUN \ -if [ "${config}" = "build_make" ] ; \ -then \ .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env; \ - .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c; \ -else \ - echo Welcome to oneDAL development, sources located in ${workdirectory} ; \ -fi + .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c; From dc6f13dc43afa132eb098e0cabad955702036815 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 15:08:46 +0100 Subject: [PATCH 30/48] adding docs env and more tests for nightly --- .github/workflows/docker-validation-nightly.yml | 12 ++++++++++-- dev/docker/onedal-dev.Dockerfile | 8 +++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index 328a000d9fd..f090f8a1e6f 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -9,12 +9,16 @@ on: paths: - .github/workflows/docker-validation-nightly.yml - dev/docker/onedal-dev.Dockerfile + - .ci/env + - .ci/scripts push: branches: - master paths: - .github/workflows/docker-validation-nightly.yml - dev/docker/onedal-dev.Dockerfile + - .ci/env + - .ci/scripts jobs: validate: @@ -25,5 +29,9 @@ jobs: uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 - name: Build docker image run: docker build . --file dev/docker/onedal-dev.Dockerfile --build-arg config=build_make --tag onedal-dev:$(date +%s) - - name: Save docker image - run: docker save --output onedal-dev.tar onedal-dev + - name: Building DAAL + run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env + - name: Building oneDAL + run: .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c + - name: Building oneDAL with bazel + run: bazel build :release diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index c3edfe30517..0fffaf03433 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -24,7 +24,7 @@ WORKDIR ${workdirectory} #Env setup RUN apt-get update && \ - apt-get -y install sudo wget gnupg git make + apt-get -y install sudo wget gnupg git make python3-setuptools doxygen # Install miniconda ENV CONDA_DIR /opt/conda @@ -55,7 +55,5 @@ RUN ./dev/download_micromkl.sh # Installing TBB dependency RUN ./dev/download_tbb.sh -# Temporary testing -RUN \ - .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env; \ - .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c; +# Installing doc build dependency +RUN pip3 install -r docs/requirements.txt From 66f6e698ac2ace01812b2e8717b4ea2e3cb88318 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 15:10:14 +0100 Subject: [PATCH 31/48] not passing param --- .github/workflows/docker-validation-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index f090f8a1e6f..b3bda7f1fef 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -28,7 +28,7 @@ jobs: - name: Checkout uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 - name: Build docker image - run: docker build . --file dev/docker/onedal-dev.Dockerfile --build-arg config=build_make --tag onedal-dev:$(date +%s) + run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:$(date +%s) - name: Building DAAL run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env - name: Building oneDAL From 515659d9389713d4d7a071b1f8a1d23577edd5bc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 15:17:12 +0100 Subject: [PATCH 32/48] switching to conda env file --- .ci/env/apt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/env/apt.sh b/.ci/env/apt.sh index 3e7230ad9ca..7a77e210c14 100755 --- a/.ci/env/apt.sh +++ b/.ci/env/apt.sh @@ -51,7 +51,7 @@ function install_dev-base { } function install_dev-base-conda { - conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 + conda env create -f environment.yml } if [ "${component}" == "dpcpp" ]; then From c746fb7eb101d81559a26e9c576fd30ebdf908a4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 15:17:37 +0100 Subject: [PATCH 33/48] adding conda env --- .ci/env/environment.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .ci/env/environment.yml diff --git a/.ci/env/environment.yml b/.ci/env/environment.yml new file mode 100644 index 00000000000..0a704f66f14 --- /dev/null +++ b/.ci/env/environment.yml @@ -0,0 +1,7 @@ +name: ci-env +channels: + - conda-forge + - intel + - defaults +dependencies: + - impi-devel=2021.10.0 From a85f06f06f4ca86bfc3c49fecfba4ff6802d6195 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 15:21:10 +0100 Subject: [PATCH 34/48] fixign path to env.yml --- .ci/env/apt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/env/apt.sh b/.ci/env/apt.sh index 7a77e210c14..420b958385d 100755 --- a/.ci/env/apt.sh +++ b/.ci/env/apt.sh @@ -51,7 +51,7 @@ function install_dev-base { } function install_dev-base-conda { - conda env create -f environment.yml + conda env create -f .ci/env/environment.yml } if [ "${component}" == "dpcpp" ]; then From 67c0bb176c43eab69b0fbe12f67eb74496ecfc1f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 16:08:13 +0100 Subject: [PATCH 35/48] drop caching version to 1 as bazel version was fixed --- .ci/pipeline/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 9d0a5456194..6191c5295e0 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -197,9 +197,9 @@ jobs: # Commit ID is added to a cache key. Caches are immutable by design, # so we always need to change a key to upload the last version # of the Bazel cache. Cache lookup is based on `restoreKeys` option. - key: '"$(BAZEL_VERSION)" | "$(Agent.OS)" | "v2" | "$(Build.SourceVersion)"' + key: '"$(BAZEL_VERSION)" | "$(Agent.OS)" | "v1" | "$(Build.SourceVersion)"' restoreKeys: | - "$(BAZEL_VERSION)" | "$(Agent.OS)" | "v2" + "$(BAZEL_VERSION)" | "$(Agent.OS)" | "v1" path: $(BAZEL_CACHE_DIR) displayName: 'bazel-cache' From aa0e7321c92872bf8426387396e0e71723e708a6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 16:18:27 +0100 Subject: [PATCH 36/48] bump version for spxinks-prompt to 1.8 --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 89116dc6e85..0115499c508 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -26,7 +26,7 @@ soupsieve==2.4.1 Sphinx==6.2.1 sphinx-book-theme==1.0.1 sphinx-notfound-page==1.0.0 -sphinx-prompt==1.7.0 +sphinx-prompt==1.8.0 Sphinx-Substitution-Extensions==2022.2.16 sphinx-tabs==3.4.1 sphinxcontrib-applehelp==1.0.4 From d4bb9b0d8adc0b8e91a62d7dd5b75ac8a9c38cb8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 16:33:28 +0100 Subject: [PATCH 37/48] revert version --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 0115499c508..89116dc6e85 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -26,7 +26,7 @@ soupsieve==2.4.1 Sphinx==6.2.1 sphinx-book-theme==1.0.1 sphinx-notfound-page==1.0.0 -sphinx-prompt==1.8.0 +sphinx-prompt==1.7.0 Sphinx-Substitution-Extensions==2022.2.16 sphinx-tabs==3.4.1 sphinxcontrib-applehelp==1.0.4 From 97703a6711e9d9ca836a414ee687bf3f6ddbd3b5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 16:51:37 +0100 Subject: [PATCH 38/48] disabling doc reqs --- dev/docker/onedal-dev.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 0fffaf03433..14c752202d2 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -56,4 +56,4 @@ RUN ./dev/download_micromkl.sh RUN ./dev/download_tbb.sh # Installing doc build dependency -RUN pip3 install -r docs/requirements.txt +#RUN pip3 install -r docs/requirements.txt From b2dadb9805bcaf92e33c019e6f4ebe18fb3d73e5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 17:17:12 +0100 Subject: [PATCH 39/48] switchign to latest tag --- .github/workflows/docker-validation-ci.yml | 2 +- .github/workflows/docker-validation-nightly.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-validation-ci.yml b/.github/workflows/docker-validation-ci.yml index e8f76a63797..3c750f8712b 100644 --- a/.github/workflows/docker-validation-ci.yml +++ b/.github/workflows/docker-validation-ci.yml @@ -16,4 +16,4 @@ jobs: - name: Checkout uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 - name: Build docker image - run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:$(date +%s) + run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index b3bda7f1fef..a9c842c1f85 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -28,7 +28,7 @@ jobs: - name: Checkout uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 - name: Build docker image - run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:$(date +%s) + run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest - name: Building DAAL run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env - name: Building oneDAL From f8a3e637ab3b3d5f171cb7193b9ecc5f757c1738 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 18:38:45 +0100 Subject: [PATCH 40/48] actually running code in docker --- .github/workflows/docker-validation-nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index a9c842c1f85..21704e6ffbf 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -32,6 +32,6 @@ jobs: - name: Building DAAL run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env - name: Building oneDAL - run: .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c + run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c - name: Building oneDAL with bazel - run: bazel build :release + run: docker run onedal-dev bazel build :release From a854d0128f75cd3bd4d0bcc55706aee86f1158f7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 19:32:58 +0100 Subject: [PATCH 41/48] merging docker commands in one --- .github/workflows/docker-validation-nightly.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index 21704e6ffbf..5325ff6302a 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -29,9 +29,9 @@ jobs: uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 - name: Build docker image run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest - - name: Building DAAL - run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env - name: Building oneDAL - run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c + run: docker run onedal-dev " \ + .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env; \ + .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c" - name: Building oneDAL with bazel run: docker run onedal-dev bazel build :release From ee816109dd43a806158abfc615f62f62265c58c4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 20:28:06 +0100 Subject: [PATCH 42/48] merging docker commands in one --- .github/workflows/docker-validation-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index 5325ff6302a..00d7f18b25f 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -30,7 +30,7 @@ jobs: - name: Build docker image run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest - name: Building oneDAL - run: docker run onedal-dev " \ + run: docker run onedal-dev sh -c " \ .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env; \ .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c" - name: Building oneDAL with bazel From e7a1e4f298f6ef8cd7c3882a4bfdb5d051900c57 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 16 Sep 2023 20:44:07 +0100 Subject: [PATCH 43/48] merging docker commands in one --- .github/workflows/docker-validation-nightly.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index 00d7f18b25f..c4416bc42a0 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -30,8 +30,6 @@ jobs: - name: Build docker image run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest - name: Building oneDAL - run: docker run onedal-dev sh -c " \ - .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env; \ - .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target oneapi_c" + run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env - name: Building oneDAL with bazel run: docker run onedal-dev bazel build :release From 934fb3560559e6b7782e528f01557617b7ee0d77 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 17 Sep 2023 10:24:49 +0100 Subject: [PATCH 44/48] installing bazel in docker image --- dev/docker/onedal-dev.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 14c752202d2..14541f4b4e9 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -44,7 +44,9 @@ RUN .ci/env/apt.sh dpcpp RUN .ci/env/apt.sh clang-format # Installing environment for bazel -RUN .ci/env/bazelisk.sh +RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 && \ + chmod 755 bazelisk-linux-amd64 && \ + mv bazelisk-linux-amd64 /usr/bin/bazel # Installing openBLAS dependency RUN .ci/env/openblas.sh From 2973ee9475edd3b9fd7ed26d0e16035144f9563d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 17 Sep 2023 17:51:29 +0100 Subject: [PATCH 45/48] adding build with OpenBLAS to validation --- .github/workflows/docker-validation-nightly.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index c4416bc42a0..023b2aafda3 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -33,3 +33,5 @@ jobs: run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env - name: Building oneDAL with bazel run: docker run onedal-dev bazel build :release + - name: Building oneDAL with OpenBLAS backend + run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --backend_config ref --conda-env ci-env From bac4594225a5301028ebaa44e4af78745635d609 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 17 Sep 2023 21:10:36 +0100 Subject: [PATCH 46/48] removing build with OpenBLAS to validation --- .github/workflows/docker-validation-nightly.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml index 023b2aafda3..c4416bc42a0 100644 --- a/.github/workflows/docker-validation-nightly.yml +++ b/.github/workflows/docker-validation-nightly.yml @@ -33,5 +33,3 @@ jobs: run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env - name: Building oneDAL with bazel run: docker run onedal-dev bazel build :release - - name: Building oneDAL with OpenBLAS backend - run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --backend_config ref --conda-env ci-env From 18740ded77450943eefd66f9e702d88576ae4dda Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 18 Sep 2023 11:02:47 +0100 Subject: [PATCH 47/48] Apply suggestions from code review Co-authored-by: Alexandra --- INSTALL.md | 6 +++--- dev/bazel/README.md | 2 +- dev/docker/README.md | 12 ++++++------ dev/docker/onedal-dev.Dockerfile | 6 ++---- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index fef8bd4fca0..989846a83a3 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -28,10 +28,10 @@ Required Software: For details, see [System Requirements for oneDAL](https://www.intel.com/content/www/us/en/developer/articles/system-requirements/system-requirements-for-oneapi-data-analytics-library.html). -## Docker development environement +## Docker Development Environment -[Docker file](https://github.com/oneapi-src/oneDAL/tree/master/dev) with oneDAL development environemnt -is avaialbe as alternative for setting up environemnt manually. +[Docker file](https://github.com/oneapi-src/oneDAL/tree/master/dev) with the oneDAL development environment +is available as an alternative to the manual setup. ## Installation Steps 1. Clone the sources from GitHub\* as follows: diff --git a/dev/bazel/README.md b/dev/bazel/README.md index 2593342d3e6..e32bc16b04f 100644 --- a/dev/bazel/README.md +++ b/dev/bazel/README.md @@ -16,7 +16,7 @@ *******************************************************************************/--> # Bazel Guide ## Install Bazel on Linux -1. Use bazelisk +1. Use Bazelisk ```sh wget -O bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 ``` diff --git a/dev/docker/README.md b/dev/docker/README.md index 9fa9472eedb..df4ba32bd63 100644 --- a/dev/docker/README.md +++ b/dev/docker/README.md @@ -15,14 +15,14 @@ * limitations under the License. *******************************************************************************/--> -# Docker dev environemnt -## How to use -This is simple docker dev environment intended for oneDAL development and build process. -It does include dependencies for building all oneDAL components through both make and bazel +# Docker Development Environment +## How To Use +There is a simple docker dev environment for the oneDAL development and build process. +It includes dependencies for building all oneDAL components with ``make`` and ``bazel`` -Note: docker setup assumes that it's executed from oneDAL checkouted repo and copy repo files inside container +Note: The docker setup assumes that it is executed from the oneDAL repo and copies repo files inside the container -Just run: +For that, run: ```sh docker run -it onedal-dev /bin/bash ``` diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile index 14541f4b4e9..36ae0702dff 100644 --- a/dev/docker/onedal-dev.Dockerfile +++ b/dev/docker/onedal-dev.Dockerfile @@ -31,7 +31,7 @@ ENV CONDA_DIR /opt/conda RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda -# Put conda in path so we can use conda activate +# Put conda in path to use conda activate ENV PATH=$CONDA_DIR/bin:$PATH # Installing environment for base development dependencies @@ -54,8 +54,6 @@ RUN .ci/env/openblas.sh # Installing MKL dependency RUN ./dev/download_micromkl.sh -# Installing TBB dependency +# Installing oneTBB dependency RUN ./dev/download_tbb.sh -# Installing doc build dependency -#RUN pip3 install -r docs/requirements.txt From 812454d56781b5a24d1e406bd93ced1272629286 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 26 Sep 2023 16:48:23 +0100 Subject: [PATCH 48/48] Apply suggestions from code review Co-authored-by: Alexander Andreev --- .ci/pipeline/ci.yml | 4 ++-- INSTALL.md | 2 +- dev/bazel/README.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 5657b13707d..6a8bf2457c4 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -185,9 +185,9 @@ jobs: BAZEL_CACHE_MAX_SIZE_KB: 4194304 # Size in kilobytes ~ 4Gb steps: - script: | + # sourcing done to set bazel version value from script source .ci/env/bazelisk.sh echo "##vso[task.setvariable variable=BAZEL_VERSION]${BAZEL_VERSION}" - #sourcing done to set bazel version value from script displayName: 'install-bazel' - script: | .ci/scripts/describe_system.sh @@ -381,7 +381,7 @@ jobs: steps: - script: | brew install dos2unix tree - conda create -n ci-env -q -y -c conda-forge python=3.9 + conda create -n ci-env -q -y -c conda-forge python=3.10 source /usr/local/miniconda/etc/profile.d/conda.sh conda activate ci-env pip install -q cpufeature diff --git a/INSTALL.md b/INSTALL.md index 989846a83a3..ef09744498b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -30,7 +30,7 @@ For details, see [System Requirements for oneDAL](https://www.intel.com/content/ ## Docker Development Environment -[Docker file](https://github.com/oneapi-src/oneDAL/tree/master/dev) with the oneDAL development environment +[Docker file](https://github.com/oneapi-src/oneDAL/tree/master/dev/docker) with the oneDAL development environment is available as an alternative to the manual setup. ## Installation Steps diff --git a/dev/bazel/README.md b/dev/bazel/README.md index e32bc16b04f..f569e2bb429 100644 --- a/dev/bazel/README.md +++ b/dev/bazel/README.md @@ -16,7 +16,7 @@ *******************************************************************************/--> # Bazel Guide ## Install Bazel on Linux -1. Use Bazelisk +1. Download Bazelisk ```sh wget -O bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 ```