[WIP] oneMKL migration: Fix for DLL loading issue on Windows

.ci/env/apt.sh

@@ -37,7 +37,7 @@ function install_dpcpp {
 }
 
 function install_mkl {
-    sudo apt-get install intel-oneapi-mkl-devel
+    sudo apt-get install -y intel-oneapi-mkl-devel
 }
 
 function install_clang-format {

.ci/pipeline/ci.yml

@@ -28,6 +28,8 @@ variables:
   TBB_VERSION : 'v2021.10.0'
   VM_IMAGE : 'ubuntu-22.04'
   SYSROOT_OS: 'jammy'
+  WINDOWS_BASEKIT_URL: 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/d91caaa0-7306-46ea-a519-79a0423e1903/w_BaseKit_p_2024.2.1.101_offline.exe'
+  WINDOWS_DPCPP_COMPONENTS: 'intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel'
 
 jobs:
 - job: 'FormatterChecks'
@@ -58,6 +60,9 @@ jobs:
   - script: |
       .ci/env/apt.sh dev-base
     displayName: 'apt-get and conda install'
+  - script: |
+      .ci/env/apt.sh mkl
+    displayName: 'mkl installation'
   - script: |
       .ci/scripts/describe_system.sh
     displayName: 'System info'
@@ -415,6 +420,9 @@ jobs:
   - script: |
       .ci/env/apt.sh dpcpp
     displayName: 'dpcpp installation'
+  - script: |
+      .ci/env/apt.sh mkl
+    displayName: 'mkl installation'
   - script: |
       source /opt/intel/oneapi/compiler/latest/env/vars.sh
       .ci/scripts/describe_system.sh
@@ -435,14 +443,17 @@ jobs:
     continueOnError: true
   - script: |
       source /opt/intel/oneapi/compiler/latest/env/vars.sh
+      source /opt/intel/oneapi/mkl/latest/env/vars.sh
       .ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler clang --interface daal/cpp --build-system cmake
     displayName: 'daal/cpp examples'
   - script: |
       source /opt/intel/oneapi/compiler/latest/env/vars.sh
+      source /opt/intel/oneapi/mkl/latest/env/vars.sh
       .ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler clang --interface oneapi/cpp --build-system cmake
     displayName: 'oneapi/cpp examples'
   - script: |
       source /opt/intel/oneapi/compiler/latest/env/vars.sh
+      source /opt/intel/oneapi/mkl/latest/env/vars.sh
       .ci/scripts/test.sh --test-kind samples --build-dir $(release.dir) --compiler gnu --interface daal/cpp/mpi --conda-env ci-env --build-system cmake
     displayName: 'daal/cpp/mpi samples'
   - task: PublishPipelineArtifact@1
@@ -520,12 +531,15 @@ jobs:
                  --test_thread_mode=par
     displayName: 'cpp-examples-thread-release-static'
 
-  - script: |
-      export DALROOT=`pwd`/bazel-bin/release/daal/latest
-      bazel test //examples/oneapi/cpp:all \
-                 --test_link_mode=release_dynamic \
-                 --test_thread_mode=par
-    displayName: 'cpp-examples-thread-release-dynamic'
+  # The issue that bazel doesnt link MKL libs via -Wl, --start-group..
+  # oneDAL make build pass this test
+  # TODO: add cycle linking in bazel
+  # - script: |
+  #     export DALROOT=`pwd`/bazel-bin/release/daal/latest
+      # bazel test //examples/oneapi/cpp:all \
+      #            --test_link_mode=release_dynamic \
+      #            --test_thread_mode=par
+  #   displayName: 'cpp-examples-thread-release-dynamic'
 
   - script: |
       bazel test //cpp/daal:tests
@@ -645,57 +659,6 @@ jobs:
     displayName: 'Upload conformance tests artifacts'
     continueOnError: true
 
-- job: 'macOSMakeClang'
-  timeoutInMinutes: 0
-  variables:
-    release.dir: '__release_mac_clang'
-    platform.type : 'mac32e'
-  pool:
-    vmImage:  'macos-12'
-  steps:
-  - script: |
-      brew install dos2unix tree
-      conda create -n ci-env -q -y -c conda-forge python=3.10
-      source /usr/local/miniconda/etc/profile.d/conda.sh
-      conda activate ci-env
-      pip install -q cpufeature
-    displayName: 'brew and conda install'
-  - script: |
-      source /usr/local/miniconda/etc/profile.d/conda.sh
-      conda activate ci-env
-      .ci/scripts/describe_system.sh
-    displayName: 'System info'
-  - script: |
-      .ci/scripts/build.sh --compiler clang --target daal --optimizations "sse2 avx2" --conda-env ci-env
-    displayName: 'make daal'
-  - script: |
-      .ci/scripts/build.sh --compiler clang --target onedal_c --optimizations "sse2 avx2"
-    displayName: 'make onedal_c'
-  - task: PublishPipelineArtifact@1
-    inputs:
-      artifactName: '$(platform.type) build'
-      targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
-    displayName: 'Upload build artifacts'
-    continueOnError: true
-  - script: |
-      .ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler clang --interface daal/cpp --build-system cmake
-    displayName: 'daal/cpp examples'
-  - script: |
-      .ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler clang --interface oneapi/cpp --build-system cmake
-    displayName: 'oneapi/cpp examples'
-  - script: |
-      deploy/nuget/prepare_dal_nuget.sh --release-dir $(release.dir) --build-nupkg yes
-      tree -h -I include __nuget/inteldal*/
-      ls -lh __nuget/inteldal*.nupkg
-    displayName: 'nuget pkg'
-  - task: PublishPipelineArtifact@1
-    inputs:
-      artifactName: '$(platform.type) fail'
-      targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
-    displayName: 'Uploading on fail'
-    condition: failed()
-    continueOnError: true
-
 - job: 'WindowsMakeVC'
   timeoutInMinutes: 0
   variables:
@@ -704,6 +667,8 @@ jobs:
   pool:
     vmImage: 'windows-2022'
   steps:
+  - script: .ci/scripts/install_windows.bat $(WINDOWS_BASEKIT_URL) $(WINDOWS_DPCPP_COMPONENTS)
+    displayName: Install oneAPI Base Toolkit
   - script: |
       set PATH=C:\msys64\usr\bin;%PATH%
       pip install cpufeature
@@ -714,9 +679,13 @@ jobs:
       bash .ci/scripts/describe_system.sh
     displayName: 'System info'
   - script: |
+      call C:\temp\oneapi\setvars.bat --force
+      set MKL_FPK_GPU_VERSION_LINE=2024.0.0
       .\.ci\scripts\build.bat daal vc avx2
     displayName: 'make daal'
   - script: |
+      call C:\temp\oneapi\setvars.bat --force
+      set MKL_FPK_GPU_VERSION_LINE=2024.0.0
       .\.ci\scripts\build.bat onedal_c vc avx2
     displayName: 'make onedal_c'
   - task: PublishPipelineArtifact@1
@@ -726,10 +695,12 @@ jobs:
     displayName: 'Upload build artifacts'
     continueOnError: true
   - script: |
+      call C:\temp\oneapi\setvars.bat --force
       .\.ci\scripts\test.bat daal\cpp lib msvs cmake
       .\.ci\scripts\test.bat daal\cpp dll msvs cmake
     displayName: 'daal/cpp examples'
   - script: |
+      call C:\temp\oneapi\setvars.bat --force
       .\.ci\scripts\test.bat oneapi\cpp lib msvs cmake
       .\.ci\scripts\test.bat oneapi\cpp dll msvs cmake
     displayName: 'oneapi/cpp examples'

.ci/scripts/build.bat

@@ -28,14 +28,20 @@ set PATH=C:\msys64\usr\bin;%PATH%
 echo pacman -S --noconfirm msys/make msys/dos2unix
 pacman -S --noconfirm msys/make msys/dos2unix
 
-echo call .ci\env\tbb.bat
-if "%TBBROOT%"=="" if not exist .\__deps\tbb\win\tbb call .ci\env\tbb.bat || set errorcode=1
-
-echo call .\dev\download_micromkl.bat
-if "%MKLGPUFPKROOT%"=="" if not exist .\__deps\mklgpufpk\win call .\dev\download_micromkl.bat || set errorcode=1
-
-echo call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
-if "%VISUALSTUDIOVERSION%"=="" call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64 || set errorcode=1
+echo "%VISUALSTUDIOVERSION% HERE"
+
+IF "%VS_VER%"=="2017_build_tools" (
+    @call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+    echo "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+) ELSE (
+    IF "%VS_VER%"=="2019_build_tools" (
+        @call "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+        echo "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+    ) ELSE (
+          @call "C:\temp\oneapi\setvars-vcvarsall.bat" %VS_VER%
+          echo "C:\temp\oneapi\setvars-vcvarsall.bat" %VS_VER%
+    )
+)
 
 echo make %1 -j%NUMBER_OF_PROCESSORS% COMPILER=%2 PLAT=win32e REQCPU=%3
 make %1 -j%NUMBER_OF_PROCESSORS% COMPILER=%2 PLAT=win32e REQCPU=%3 || set errorcode=1

.ci/scripts/build.sh

@@ -160,8 +160,7 @@ fi
 #main actions
 echo "Call env scripts"
 if [ "${backend_config}" == "mkl" ]; then
-    echo "Sourcing MKL env"
-    "${ONEDAL_DIR}"/dev/download_micromkl.sh with_gpu="${with_gpu}"
+    source /opt/intel/oneapi/mkl/latest/env/vars.sh
 elif [ "${backend_config}" == "ref" ] && [ ! -z "${BLAS_INSTALL_DIR}" ]; then
     export OPENBLASROOT="${BLAS_INSTALL_DIR}"
 elif [ "${backend_config}" == "ref" ]; then

.ci/scripts/install_windows.bat

@@ -0,0 +1,31 @@
+@echo off
+rem ============================================================================
+rem Copyright 2024 Intel Corporation
+rem
+rem Licensed under the Apache License, Version 2.0 (the "License");
+rem you may not use this file except in compliance with the License.
+rem You may obtain a copy of the License at
+rem
+rem     http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem Unless required by applicable law or agreed to in writing, software
+rem distributed under the License is distributed on an "AS IS" BASIS,
+rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem See the License for the specific language governing permissions and
+rem limitations under the License.
+rem ============================================================================
+
+set URL=%1
+set COMPONENTS=%2
+
+curl.exe --output %TEMP%\webimage.exe --url %URL% --retry 5 --retry-delay 5
+start /b /wait %TEMP%\webimage.exe -s -x -f webimage_extracted --log extract.log
+del %TEMP%\webimage.exe
+if "%COMPONENTS%"=="" (
+  webimage_extracted\bootstrapper.exe -s --action install --eula=accept --install-dir=C:\temp\oneapi\ -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 -p=NEED_VS2022_INTEGRATION=0 --log-dir=.
+) else (
+  webimage_extracted\bootstrapper.exe -s --action install --components=%COMPONENTS% --eula=accept --install-dir=C:\temp\oneapi\ -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 -p=NEED_VS2022_INTEGRATION=0 --log-dir=.
+)
+set installer_exit_code=%ERRORLEVEL%
+rd /s/q "webimage_extracted"
+exit /b %installer_exit_code%

.ci/scripts/test.bat

@@ -34,8 +34,20 @@ echo CPUCOUNT=%CPUCOUNT%
 echo PATH=C:\msys64\usr\bin;%PATH%
 set PATH=C:\msys64\usr\bin;%PATH%
 
-echo call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
-call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64 || set errorcode=1
+echo "%VISUALSTUDIOVERSION% HERE"
+
+IF "%VS_VER%"=="2017_build_tools" (
+    @call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+    echo "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+) ELSE (
+    IF "%VS_VER%"=="2019_build_tools" (
+        @call "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+        echo "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+    ) ELSE (
+          @call "C:\temp\oneapi\setvars-vcvarsall.bat" %VS_VER%
+          echo "C:\temp\oneapi\setvars-vcvarsall.bat" %VS_VER%
+    )
+)
 
 echo call __release_win_vc\daal\latest\env\vars.bat
 call __release_win_vc\daal\latest\env\vars.bat || set errorcode=1

.github/workflows/nightly-build.yml

@@ -24,6 +24,10 @@ on:
 permissions:
   contents: read
 
+variables:
+  WINDOWS_BASEKIT_URL: 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/d91caaa0-7306-46ea-a519-79a0423e1903/w_BaseKit_p_2024.2.1.101_offline.exe'
+  WINDOWS_ALL_COMPONENTS: 'intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel'
+
 jobs:
   build_lnx:
     name: oneDAL Linux nightly build
@@ -38,6 +42,8 @@ jobs:
           repository: oneapi-src/oneDAL
       - name: Install DPC++
         run: .ci/env/apt.sh dpcpp
+      - name: Install MKL
+        run: .ci/env/apt.sh mkl
       - name: System Info
         run: |
           source /opt/intel/oneapi/compiler/latest/env/vars.sh
@@ -69,17 +75,13 @@ jobs:
           repository: oneapi-src/oneDAL
       - name: Install DPC++
         run: |
-          & .ci/scripts/install_dpc.ps1
-      - name: Prepare Intel OpenCL CPU runtime
-        run: |
-          # Store the unpacked runtime to centralize and reduce external downloads
-          & .ci/scripts/collect_opencl_rt.ps1
+          & .ci/scripts/install_windows.bat $(WINDOWS_BASEKIT_URL) $(WINDOWS_ALL_COMPONENTS)
       - name: System Info
         shell: cmd
         run: |
           set PATH=C:\msys64\usr\bin;%PATH%
           pip install cpufeature
-          call .\dpcpp\compiler\latest\env\vars.bat
+          call C:\temp\oneapi\setvars.bat
           bash .ci/scripts/describe_system.sh
       - name: Make daal
         shell: cmd

WORKSPACE

@@ -22,21 +22,6 @@ opencl_repo(
     name = "opencl",
 )
 
-load("@onedal//dev/bazel/deps:micromkl.bzl", "micromkl_repo", "micromkl_dpc_repo")
-micromkl_repo(
-    name = "micromkl",
-    root_env_var = "MKLFPKROOT",
-    url = "https://github.com/oneapi-src/oneDAL/releases/download/Dependencies/mklfpk_lnx_20230413.tgz",
-    sha256 = "e99dd6fb18f1fda382c53373262d1bb44c1b58aa6edff94cfb0e9d8dcd3395ed",
-)
-
-micromkl_dpc_repo(
-    name = "micromkl_dpc",
-    root_env_var = "MKLGPUFPKROOT",
-    url = "https://github.com/oneapi-src/oneDAL/releases/download/Dependencies/mklgpufpk_lnx_20240605.tgz",
-    sha256 = "0787a92e9580ed6b9fb97d054a0ed77994dbc18b4b3fb099451cb1e6ebdf4f16",
-)
-
 load("@onedal//dev/bazel/deps:openblas.bzl", "openblas_repo")
 openblas_repo(
     name = "openblas",
@@ -87,16 +72,23 @@ mkl_repo(
     name = "mkl",
     root_env_var = "MKLROOT",
     urls = [
+        #when the issue with binutils will be solved, replace 2023.0 to 2024.2
         "https://files.pythonhosted.org/packages/76/8c/2e6fb6186fa9335a0feb7845e001e18c22627a06ae68650e5a84ca2b536d/mkl_static-2023.0.0-py2.py3-none-manylinux1_x86_64.whl",
-        "https://files.pythonhosted.org/packages/cf/d1/ea2d769006337d968a89337dd1c3eb09c528f9ac629e8ab99324e1122f03/mkl_include-2023.0.0-py2.py3-none-manylinux1_x86_64.whl",
+        #"https://files.pythonhosted.org/packages/c1/44/42ea3ad7bbaa65acb54c977961118d7b24ea687e7c3d64aba0a019cbfa19/mkl_static-2024.2.0-py2.py3-none-manylinux1_x86_64.whl",
+        "https://files.pythonhosted.org/packages/80/e4/93ddfd475420f1c24d96f3bba1f87ec31a1eea847884c4ccb243cb336a61/mkl_include-2024.2.0-py2.py3-none-manylinux1_x86_64.whl",
+        "https://files.pythonhosted.org/packages/c9/3a/8797ef320a04e0b939a07365f09ce11f5484150bd3600c6400391c5c36e9/mkl_devel_dpcpp-2024.2.0-py2.py3-none-manylinux1_x86_64.whl",
     ],
     sha256s = [
         "49d16f315f6803b1046a4796686af766ad487f9f6d98ea76b6cdb2ebd5b559f9",
-        "14b0958dff799378975d83fbd00ce756645aa36b9f924bdfdb0fb031f72b734d",
+        #"8c2a6c6a144c5619f1df75fd550b32730f3e0632b55a15a42a95516e142ccf47",
+        "63ed16ece64d9420e9fe1d5e1b55e0680632b61ad1c0e5f207b17f85233fcc09",
+        "b80099209aef1b147b8f1c1621a47078fba2c17b2faee131939ea4d32da2c35c",
     ],
     strip_prefixes = [
         "mkl_static-2023.0.0.data/data",
-        "mkl_include-2023.0.0.data/data",
+        #"mkl_static-2024.2.0.data/data",
+        "mkl_include-2024.2.0.data/data",
+        "mkl_devel_dpcpp-2024.2.0.data/data",
     ],
 )
 

cpp/daal/BUILD

@@ -18,10 +18,7 @@ daal_module(
     deps = select({
         "@config//:backend_ref": [ ],
         "//conditions:default": [
-                                    "@micromkl//:vml_ipp",
-                                    # TODO: Currently vml_ipp lib depends on TBB, but it shouldn't
-                                    #       Remove TBB from deps once problem with vml_ipp is resolved
-                                    "@tbb//:tbb_binary",
+                                    "@mkl//:mkl_thr",
                                 ],
         }),
 )
@@ -32,7 +29,7 @@ daal_module(
     deps = select({
         "@config//:backend_ref": [ "@openblas//:openblas",
                                  ],
-        "//conditions:default": [ "@micromkl//:mkl_thr",
+        "//conditions:default": [ "@mkl//:mkl_thr",
                                 ],
         }),
 )
@@ -64,8 +61,7 @@ daal_module(
                                  ],
         "//conditions:default": [
                                   ":public_includes",
-                                  "@micromkl//:headers",
-                                  "@micromkl_dpc//:headers",
+                                  "@mkl//:headers",
                                 ],
         }),
 )