From 00482c2f363a4fe67b34c03c04fa2293b14f3db5 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Wed, 6 Nov 2024 13:39:27 +0100 Subject: [PATCH] [CI] upgrade GitHub actions to be more robust and upgrade dpnp/dpctl in CI (#2147) * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update describe_system.sh * Update ci.yml * Update ci.yml * Update activate_components.sh * Update activate_components.bat * Update ci.yml --- .ci/scripts/describe_system.sh | 5 +++ .github/scripts/activate_components.bat | 6 ++-- .github/scripts/activate_components.sh | 9 ++++-- .github/workflows/ci.yml | 42 +++++++++++++++---------- 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.ci/scripts/describe_system.sh b/.ci/scripts/describe_system.sh index 0372e19d04..a30f552e1e 100755 --- a/.ci/scripts/describe_system.sh +++ b/.ci/scripts/describe_system.sh @@ -57,3 +57,8 @@ if [ -x "$(command -v icpx)" ]; then echo "ICPX:" icpx --version fi +echo +# SYCL devices +if [ -x "$(command -v sycl-ls)" ]; then + sycl-ls +fi diff --git a/.github/scripts/activate_components.bat b/.github/scripts/activate_components.bat index 0f1d0d5453..a0acf1fcd4 100644 --- a/.github/scripts/activate_components.bat +++ b/.github/scripts/activate_components.bat @@ -19,9 +19,7 @@ rem %1 - dpcpp activate flag rem prepare vc call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64 -rem prepare icx only if no parameter is given. -if "%1"=="" call .\oneapi\compiler\latest\env\vars.bat -rem prepare tbb -call .\oneapi\tbb\latest\env\vars.bat +rem prepare only TBB from oneAPI if a parameter is given. +IF "%1"=="" (call .\oneapi\setvars.bat) ELSE (call .\oneapi\tbb\latest\env\vars.bat) rem prepare oneDAL call .\__release_win\daal\latest\env\vars.bat diff --git a/.github/scripts/activate_components.sh b/.github/scripts/activate_components.sh index abc37ec7c0..62daf3a34d 100644 --- a/.github/scripts/activate_components.sh +++ b/.github/scripts/activate_components.sh @@ -15,7 +15,10 @@ # limitations under the License. #=============================================================================== -# if any parameter is given, then do not source the compiler -if [ $# -eq 0 ]; then source /opt/intel/oneapi/compiler/latest/env/vars.sh; fi # prepare icpx -source /opt/intel/oneapi/tbb/latest/env/vars.sh # prepare tbb +# if any parameter is given then only source TBB from the oneAPI install +if [ $# -eq 0 ]; then + source /opt/intel/oneapi/setvars.sh +else + source /opt/intel/oneapi/tbb/latest/env/vars.sh # prepare tbb +fi source ./__release_lnx/daal/latest/env/vars.sh # prepare oneDAL diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e709f12249..dd4ccf6c74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,8 +28,8 @@ concurrency: cancel-in-progress: true env: - DPCTL_VERSION: 0.17.0 - DPNP_VERSION: 0.15.0 + DPCTL_VERSION: 0.18.1 + DPNP_VERSION: 0.16.0 DPCTL_PY_VERSIONS: '3.9\|3.11' jobs: @@ -51,11 +51,6 @@ jobs: steps: - name: Checkout Scikit-learn-intelex uses: actions/checkout@v4 - - name: Checkout oneDAL - uses: actions/checkout@v4 - with: - repository: oneapi-src/oneDAL - path: oneDAL - name: Install Python uses: actions/setup-python@v5 with: @@ -65,7 +60,8 @@ jobs: run: | OTHER_REPO="oneapi-src/oneDAL" WF_NAME="Nightly-build" - RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId --status success --jq .[0].databaseId` + JQ_QUERY='map(select(.event == "workflow_dispatch" or .event == "schedule")) | .[0].databaseId' + RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId,event --status success --jq "${JQ_QUERY}"` echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}" echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT" env: @@ -78,6 +74,14 @@ jobs: repository: oneapi-src/oneDAL run-id: ${{ steps.get-run-id.outputs.run-id }} path: ./__release_lnx + - name: Download oneDAL environment artifact + uses: actions/download-artifact@v4 + with: + name: oneDAL_env + github-token: ${{ github.token }} + repository: oneapi-src/oneDAL + run-id: ${{ steps.get-run-id.outputs.run-id }} + path: .ci/env - name: Set Environment Variables id: set-env run: | @@ -92,9 +96,11 @@ jobs: - name: dpcpp installation run: | # This CI system yields oneAPI dependencies from the oneDAL repository - bash ./oneDAL/.ci/env/apt.sh dpcpp + bash .ci/env/apt.sh dpcpp - name: describe system - run: bash .ci/scripts/describe_system.sh + run: | + source /opt/intel/oneapi/setvars.sh + bash .ci/scripts/describe_system.sh - name: Install develop requirements run: | python -m venv venv @@ -125,13 +131,15 @@ jobs: run: | source venv/bin/activate source .github/scripts/activate_components.sh - bash .ci/scripts/run_sklearn_tests.sh + if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi + bash .ci/scripts/run_sklearn_tests.sh $CPU - name: Sklearn testing [preview] run: | source venv/bin/activate source .github/scripts/activate_components.sh + if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi export SKLEARNEX_PREVIEW='YES' - bash .ci/scripts/run_sklearn_tests.sh + bash .ci/scripts/run_sklearn_tests.sh $CPU sklearn_win: strategy: @@ -161,7 +169,8 @@ jobs: run: | OTHER_REPO="oneapi-src/oneDAL" WF_NAME="Nightly-build" - RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId --status success --jq .[0].databaseId` + JQ_QUERY='map(select(.event == "workflow_dispatch" or .event == "schedule")) | .[0].databaseId' + RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId,event --status success --jq "${JQ_QUERY}"` echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}" echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT" env: @@ -224,7 +233,6 @@ jobs: run: | call .\venv\Scripts\activate.bat call .\oneapi\setvars.bat - call .\oneapi\compiler\latest\bin\sycl-ls.exe bash .ci/scripts/describe_system.sh - name: Build daal4py/sklearnex shell: cmd @@ -258,11 +266,13 @@ jobs: run: | call .\venv\Scripts\activate.bat call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }} - bash .ci/scripts/run_sklearn_tests.sh + if "${{ steps.set-env.outputs.DPCFLAG }}"=="" set CPU=cpu + bash .ci/scripts/run_sklearn_tests.sh %CPU% - name: Sklearn testing [preview] shell: cmd run: | call .\venv\Scripts\activate.bat call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }} + if "${{ steps.set-env.outputs.DPCFLAG }}"=="" set CPU=cpu set SKLEARNEX_PREVIEW=YES - bash .ci/scripts/run_sklearn_tests.sh + bash .ci/scripts/run_sklearn_tests.sh %CPU%