Merge branch 'oneapi-src:main' into dev/asolovev_cpath_fix

.ci/env/apt.sh

@@ -32,7 +32,7 @@ function add_repo {
 }
 
 function install_dpcpp {
-    sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-2024.1
+    sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-2024.2
     sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd'
 }
 
@@ -63,10 +63,33 @@ function install_qemu_emulation_apt {
 }
 
 function install_qemu_emulation_deb {
-    qemu_deb=qemu-user-static_8.2.2+ds-2+b1_amd64.deb
-    wget http://ftp.de.debian.org/debian/pool/main/q/qemu/${qemu_deb}
-    sudo dpkg -i ${qemu_deb}
+    set +e
+
+    versions=(9.0.2 9.0.1 8.2.4)
+    found_version=""
+    for version in ${versions[@]}; do
+        qemu_deb="qemu-user-static_${version}+ds-1_amd64.deb"
+        echo "Checking for http://ftp.debian.org/debian/pool/main/q/qemu/${qemu_deb}"
+        if wget -q --method=HEAD http://ftp.debian.org/debian/pool/main/q/qemu/${qemu_deb} &> /dev/null;
+        then
+            echo "Found qemu version ${version}"
+            found_version=${qemu_deb}
+            break
+        fi
+    done
+
+    set -eo pipefail
+    if [[ -z "${found_version}" ]] ; then
+        # If nothing is found, error out and fail
+        echo "None of the requested qemu versions ${versions[*]} are available."
+        false
+    fi
+
+    wget http://ftp.debian.org/debian/pool/main/q/qemu/${found_version}
+    sudo dpkg -i ${found_version}
+
     sudo systemctl restart systemd-binfmt.service
+    set +eo pipefail
 }
 
 function install_llvm_version {

.ci/env/environment.yml

@@ -1,7 +1,6 @@
 name: ci-env
 channels:
   - conda-forge
-  - intel
   - defaults
 dependencies:
-  - impi-devel=2021.10.0
+  - impi-devel=2021.12.0

.ci/env/openblas.sh

@@ -37,6 +37,7 @@ show_help() {
 --prefix:The path where OpenBLAS will be installed
 --version:The version of OpenBLAS to install. This is a git reference from the OpenBLAS repo, and defaults to ${BLAS_DEFAULT_VERSION}
 --sysroot:If cross-compiling with LLVM, determines the location of the target architecture sysroot
+--ilp64 <on/off>: whether or not to use the ILP64 build
 '
 }
 
@@ -74,6 +75,9 @@ while [[ $# -gt 0 ]]; do
         --sysroot)
         sysroot="$2"
         shift;;
+        --ilp64)
+        ilp64=on
+        shift;;
         --help)
         show_help
         exit 0
@@ -89,6 +93,7 @@ done
 target=${target:-ARMV8}
 host_compiler=${host_compiler:-gcc}
 compiler=${compiler:-aarch64-linux-gnu-gcc}
+openblas_ilp64=${ilp64:-on}
 
 target_arch=${target_arch:-$(uname -m)}
 OPENBLAS_DEFAULT_PREFIX="${ONEDAL_DIR}/__deps/openblas_${target_arch}"
@@ -160,6 +165,9 @@ pushd "${blas_src_dir}"
         USE_THREAD=0
         USE_LOCKING=1)
   fi
+  if [ "${openblas_ilp64}" == "on" ]; then
+      make_options+=( 'BINARY=64' 'INTERFACE64=1' )
+  fi
   # Clean
   echo make "${make_options[@]}" clean
   make "${make_options[@]}" clean

.ci/pipeline/ci.yml

@@ -114,7 +114,7 @@ jobs:
     displayName: 'System info'
   - task: Cache@2
     inputs:
-      key: '"gcc" | "aarch64" | "openblas" | "$(OPENBLAS_VERSION)" | "$(VM_IMAGE)"'
+      key: '"gcc" | "aarch64" | "openblas" | "$(OPENBLAS_VERSION)" | "$(VM_IMAGE)" | "ILP64"'
       path: $(OPENBLAS_CACHE_DIR)
       cacheHitVar: OPENBLAS_RESTORED
   - script: |
@@ -201,7 +201,7 @@ jobs:
     displayName: 'System info'
   - task: Cache@2
     inputs:
-      key: '"clang" | "18" | "aarch64" | "openblas" | "$(OPENBLAS_VERSION)" | "$(VM_IMAGE)"'
+      key: '"clang" | "18" | "aarch64" | "openblas" | "$(OPENBLAS_VERSION)" | "$(VM_IMAGE)" | "ILP64"'
       path: $(OPENBLAS_CACHE_DIR)
       cacheHitVar: OPENBLAS_RESTORED
   - script: |
@@ -293,7 +293,7 @@ jobs:
     displayName: 'System info'
   - task: Cache@2
     inputs:
-      key: '"clang" | "riscv64" | "openblas" | "$(OPENBLAS_VERSION)"'
+      key: '"clang" | "riscv64" | "openblas" | "$(OPENBLAS_VERSION)" | "ILP64"'
       path: $(OPENBLAS_CACHE_DIR)
       cacheHitVar: OPENBLAS_RESTORED
   - script: |
@@ -363,7 +363,7 @@ jobs:
     displayName: 'System info'
   - task: Cache@2
     inputs:
-      key: '"gnu" | "x86_64" | "openblas" | "$(OPENBLAS_VERSION)" | "$(VM_IMAGE)"'
+      key: '"gcc" | "x86_64" | "openblas" | "$(OPENBLAS_VERSION)" | "$(VM_IMAGE)" | "ILP64"'
       path: $(OPENBLAS_CACHE_DIR)
       cacheHitVar: OPENBLAS_RESTORED
   - script: |
@@ -558,7 +558,7 @@ jobs:
       fi
     displayName: 'bazel-cache-limit'
 
-- job: LinuxDaal4py
+- job: LinuxSklearnex
   dependsOn: LinuxMakeGNU_MKL
   timeoutInMinutes: 0
   variables:
@@ -583,57 +583,50 @@ jobs:
     displayName: 'System info'
   - script: |
       conda update -y -q conda
-      conda create -q -y -n CB -c intel python=$(python.version) tbb mpich
+      conda create -q -y -n CB -c conda-forge python=$(python.version) tbb mpich
     displayName: 'Conda create'
   - script: |
-      git clone https://github.com/intel/scikit-learn-intelex.git daal4py
-    displayName: Clone daal4py
+      git clone https://github.com/intel/scikit-learn-intelex.git sklearnex
+    displayName: Clone sklearnex
   - script: |
       source /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
-      pip install -r daal4py/dependencies-dev
-      pip install -r daal4py/requirements-test.txt
+      pip install -r sklearnex/dependencies-dev
+      pip install -r sklearnex/requirements-test.txt
     displayName: Create python environment
   - script: |
       source /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
       export DALROOT=$(Pipeline.Workspace)/daal/latest
       source ${DALROOT}/env/vars.sh
-      cd daal4py
+      cd sklearnex
       export PYTHON=python
       ./conda-recipe/build.sh
-    displayName: daal4py build
+    displayName: sklearnex build
   - task: PublishPipelineArtifact@1
     inputs:
-      artifactName: '$(platform.type) daal4py build'
-      targetPath: '$(Build.Repository.LocalPath)/daal4py'
-    displayName: 'Upload daal4py build artifacts'
+      artifactName: '$(platform.type) sklearnex build'
+      targetPath: '$(Build.Repository.LocalPath)/sklearnex'
+    displayName: 'Upload sklearnex build artifacts'
     continueOnError: true
-  - script: |
-      . /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate CB
-      export DALROOT=$(Pipeline.Workspace)/daal/latest
-      cd daal4py
-      python setup_sklearnex.py install --single-version-externally-managed --record=record.txt
-    displayName: sklearnex build
   - script: |
       source /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
       source $(Pipeline.Workspace)/daal/latest/env/vars.sh
-      ./daal4py/conda-recipe/run_test.sh
+      ./sklearnex/conda-recipe/run_test.sh
     timeoutInMinutes: 15
-    displayName: daal4py test
+    displayName: sklearnex test
   - script: |
       source /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
       source $(Pipeline.Workspace)/daal/latest/env/vars.sh
       ret_code=0
-      python -m daal4py daal4py/tests/run_examples.py
+      python -m sklearnex sklearnex/tests/run_examples.py
       ret_code=$(($ret_code + $?))
-      python -m daal4py daal4py/tests/daal4py/sycl/sklearn_sycl.py
+      python -m sklearnex sklearnex/tests/daal4py/sycl/sklearn_sycl.py
       ret_code=$(($ret_code + $?))
       exit $ret_code
-    displayName: daal4py examples
+    displayName: sklearnex examples
   - script: |
       source /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB

.ci/scripts/build.bat

@@ -19,8 +19,8 @@ rem %1 - Make target
 rem %2 - Compiler
 rem %3 - Instruction set
 
-for /f "tokens=*" %%i in ('python -c "from multiprocessing import cpu_count; print(cpu_count())"') do set CPUCOUNT=%%i
-echo CPUCOUNT=%CPUCOUNT%
+set errorcode=0
+echo CPUCOUNT=%NUMBER_OF_PROCESSORS%
 
 echo PATH=C:\msys64\usr\bin;%PATH%
 set PATH=C:\msys64\usr\bin;%PATH%
@@ -29,15 +29,16 @@ echo pacman -S --noconfirm msys/make msys/dos2unix
 pacman -S --noconfirm msys/make msys/dos2unix
 
 echo call .ci\env\tbb.bat
-call .ci\env\tbb.bat
+if "%TBBROOT%"=="" if not exist .\__deps\tbb\win\tbb call .ci\env\tbb.bat || set errorcode=1
 
 echo call .\dev\download_micromkl.bat
-call .\dev\download_micromkl.bat
+if "%MKLGPUFPKROOT%"=="" if not exist .\__deps\mklgpufpk\win call .\dev\download_micromkl.bat || set errorcode=1
 
 echo call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
-call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
+if "%VISUALSTUDIOVERSION%"=="" call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64 || set errorcode=1
 
-echo make %1 -j%CPUCOUNT% COMPILER=%2 PLAT=win32e REQCPU=%3
-make %1 -j%CPUCOUNT% COMPILER=%2 PLAT=win32e REQCPU=%3
+echo make %1 -j%NUMBER_OF_PROCESSORS% COMPILER=%2 PLAT=win32e REQCPU=%3
+make %1 -j%NUMBER_OF_PROCESSORS% COMPILER=%2 PLAT=win32e REQCPU=%3 || set errorcode=1
 
-cmake -DINSTALL_DIR=__release_win_vc\daal\latest\lib\cmake\oneDAL -DARCH_DIR=intel64 -P cmake\scripts\generate_config.cmake
+cmake -DINSTALL_DIR=__release_win_vc\daal\latest\lib\cmake\oneDAL -DARCH_DIR=intel64 -P cmake\scripts\generate_config.cmake || set errorcode=1
+EXIT /B %errorcode%

.ci/scripts/test.bat

@@ -35,10 +35,10 @@ echo PATH=C:\msys64\usr\bin;%PATH%
 set PATH=C:\msys64\usr\bin;%PATH%
 
 echo call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
-call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
+call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64 || set errorcode=1
 
 echo call __release_win_vc\daal\latest\env\vars.bat
-call __release_win_vc\daal\latest\env\vars.bat
+call __release_win_vc\daal\latest\env\vars.bat || set errorcode=1
 
 echo set LIB=%~dp0..\..\__release_win_vc\tbb\latest\lib\intel64\vc_mt;%LIB%
 set LIB=%~dp0..\..\__release_win_vc\tbb\latest\lib\intel64\vc_mt;%LIB%
@@ -69,9 +69,9 @@ if "%build_system%"=="cmake" (
 
     set results_dir=_cmake_results\intel_intel64_%cmake_link_mode_short%\Release
     echo cmake -B Build -S . -DONEDAL_LINK=%cmake_link_mode% -DTBB_DIR=%TBB_DIR%
-    cmake -B Build -S . -DONEDAL_LINK=%cmake_link_mode% -DTBB_DIR=%TBB_DIR%
+    cmake -B Build -S . -DONEDAL_LINK=%cmake_link_mode% -DTBB_DIR=%TBB_DIR% || set errorcode=1
     set solution_name=%examples:\=_%
-    msbuild.exe "Build\!solution_name!_examples.sln" /p:Configuration=Release
+    msbuild.exe "Build\!solution_name!_examples.sln" /p:Configuration=Release || set errorcode=1
 
     for /f "delims=." %%F in ('dir /B !results_dir!\*.exe 2^> nul') do (
         set example=%%F
@@ -100,3 +100,4 @@ if "%build_system%"=="cmake" (
     if "%examples%"=="daal\cpp" nmake %linking% compiler=%compiler%
     if "%examples%"=="oneapi\cpp" nmake %linking% compiler=%compiler%
 )
+EXIT /B %errorcode%

.github/CODEOWNERS

@@ -1,6 +1,6 @@
 # Documentation owners and reviewers
-/docs/         @Vika-F @maria-Petrova @Alexsandruss @aepanchi
-*.md           @Vika-F @maria-Petrova @Alexsandruss @aepanchi
+/docs/         @Vika-F @maria-Petrova @Alexsandruss @bdmoore1
+*.md           @Vika-F @maria-Petrova @Alexsandruss @bdmoore1
 
 # TTP files
 third-party*   @maria-Petrova
@@ -22,7 +22,7 @@ deploy/        @Alexsandruss @napetrov @homksei @ahuber21 @ethanglaser
 dev/           @Alexsandruss @napetrov @homksei @ahuber21 @ethanglaser 
 
 # C++ code
-cpp/           @Alexsandruss @samir-nasibli @KulikovNikita @Alexandr-Solovev
+cpp/           @Alexsandruss @samir-nasibli @Alexandr-Solovev
 
 # Tree based methods
 dtrees         @razdoburdin @ahuber21 @avolkov-intel @icfaust

.github/workflows/docker-validation-ci.yml

@@ -17,6 +17,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - name: Build docker image
         run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest

.github/workflows/docker-validation-nightly.yml

@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - name: Build docker image
         run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest
       - name: Building oneDAL

.github/workflows/openssf-scorecard.yml

@@ -0,0 +1,38 @@
+name: OpenSSF Scorecard
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '0 21 * * 5'
+  push:
+    branches: [ "main" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    if: github.repository == 'oneapi-src/oneDAL'
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          publish_results: true

.github/workflows/renovate-validation.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - name: Validate
         uses: suzuki-shunsuke/github-action-renovate-config-validator@v1.0.1
         with:

.github/workflows/slack-pr-notification.yml

@@ -0,0 +1,27 @@
+name: Slack PR Notification
+on:
+  # use pull_request_target to run on PRs from forks and have access to secrets
+  pull_request_target:
+    types: [labeled]
+
+env:
+  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+  channel: "onedal"
+
+permissions:
+  pull-requests: read
+
+jobs:
+  rfc:
+    name: RFC Notification
+    runs-on: ubuntu-latest
+    # Trigger when labeling a PR with "RFC"
+    if: |
+      github.event.action == 'labeled' &&
+      contains(toJson(github.event.pull_request.labels.*.name), '"RFC"')
+    steps:
+    - name: Notify Slack
+      uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
+      with:
+        channel-id: ${{ env.channel }}
+        slack-message: "${{ github.actor }} posted a RFC: ${{ github.event.pull_request.title }}. URL: ${{ github.event.pull_request.html_url }}"