[DO NOT MERGE] Test PR for new benchmarking workflow

.github/workflows/sycl-benchmark.yml

@@ -0,0 +1,215 @@
+name: Run benchmarks (using sycl-bench)
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        type: string
+        required: True
+      image:
+        type: string
+        required: True
+      image_options:
+        type: string
+        required: True
+
+      sycl_toolchain_artifact:
+        type: string
+        default: ''
+        required: False
+      sycl_toolchain_archive:
+        type: string
+        default: ''
+        required: False
+      sycl_toolchain_decompress_command:
+        type: string
+        default: ''
+        required: False
+
+  workflow_dispatch:
+    inputs:
+      runner:
+        type: choice
+        options:
+          - '["Linux", "gen12"]'
+          - '["amdgpu"]'
+          - '["Linux", "arc"]'
+          - '["cts-cpu"]'
+      image:
+        description: |
+          Use option ending with ":build" for AMDGPU, ":latest" for the rest.
+        type: choice
+        options:
+          - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:latest'
+          - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build'
+      image_options:
+        description: |
+          Use option with "--device=/dev/kfd" for AMDGPU, without it for the rest.
+        type: choice
+        options:
+          - '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN'
+          - '-u 1001 --device=/dev/dri --device=/dev/kfd --privileged --cap-add SYS_ADMIN'
+
+      sycl_toolchain_artifact:
+        type: string
+        default: ''
+        required: False
+      sycl_toolchain_archive:
+        type: string
+        default: ''
+        required: False
+      sycl_toolchain_decompress_command:
+        type: string
+        default: ''
+        required: False
+
+
+permissions:
+  contents: read
+
+jobs:
+  run:
+    name: Run benchmarks using sycl-bench
+    runs-on: ${{ fromJSON(inputs.runner || '["Linux", "gen12"]') }}
+    container:
+      image: ${{ inputs.image }}
+      options: ${{ inputs.image_options }}
+    steps:
+    - name: Download SYCL toolchain
+      if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run'
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.sycl_toolchain_artifact }}
+    - name: Debug prints [workflow_run]
+      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
+      run: |
+        pwd
+        ls
+    - name: Download SYCL toolchain [workflow_run]
+      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const name = '${{ inputs.sycl_toolchain_artifact }}'
+          let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+             owner: context.repo.owner,
+             repo: context.repo.repo,
+             run_id: context.payload.workflow_run.id,
+          });
+          let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+            return artifact.name == name
+          })[0];
+          let download = await github.rest.actions.downloadArtifact({
+             owner: context.repo.owner,
+             repo: context.repo.repo,
+             artifact_id: matchArtifact.id,
+             archive_format: 'zip',
+          });
+          let fs = require('fs');
+          fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/` + name + '.zip', Buffer.from(download.data));
+    - name: Unzip artifact [workflow_run]
+      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
+      run: |
+        pwd
+        ls
+        unzip ${{ inputs.sycl_toolchain_artifact }}.zip
+        rm ${{ inputs.sycl_toolchain_artifact }}.zip
+    - name: Extract/Setup SYCL toolchain
+      if: inputs.sycl_toolchain_artifact != ''
+      run: |
+        mkdir toolchain
+        tar -I '${{ inputs.sycl_toolchain_decompress_command }}' -xf ${{ inputs.sycl_toolchain_archive }} -C toolchain
+        rm -f ${{ inputs.sycl_toolchain_archive }}
+        echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV
+        echo SYCL_ARTIFACT_PATH="$PWD/toolchain/" >> $GITHUB_ENV
+        echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV
+    - uses: actions/checkout@v4
+      with:
+        repository: 'ianayl/sycl-bench'
+        clean: false   # Prevents actions/checkout@v4 from deleting the extracted artifact
+        path: './sycl-bench/'
+    - name: Build sycl-bench
+      run: |
+        cd ./sycl-bench
+        cmake -DSYCL_IMPL=dpcpp -DCMAKE_CXX_COMPILER=$SYCL_ARTIFACT_PATH/bin/clang++ -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=./bin -S . -B ./build &&
+        cmake --build ./build
+        cd -
+        echo LD_LIBRARY_PATH=$SYCL_ARTIFACT_PATH/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV
+    - name: Run sycl-bench tests
+      id: run-benchmarks
+      run: |
+        # Determine the index of a column in a CSV given its title
+        # Usage: get_csv_col_index <benchmark output .csv file> <column name>
+        get_csv_col_index() {
+            tmp_csv_col_i="$(cat "$1" | head -n 1 | grep -o "^.*$2," | grep -o ',' | wc -l)"
+        }
+
+        # Usage: print_bench_res <benchmark output .csv file> <benchmark status code> <summary file>
+        print_bench_res() {
+            if [ ! -s $1 ]; then
+                printf "NO OUTPUT! (Status $2)\n" | tee -a $3
+                return  # Do not proceed if file is empty
+            fi
+
+            get_csv_col_index $1 run-time-mean
+            tmp_run_time_mean_i=$tmp_csv_col_i
+            get_csv_col_index $1 run-time-median
+            tmp_run_time_median_i=$tmp_csv_col_i
+            get_csv_col_index $1 run-time-throughput
+            tmp_run_time_throughput_i=$tmp_csv_col_i
+
+            # `sycl-bench` output seems to like inserting the header multiple times.
+            # Here we cache the header to make sure it prints only once:
+            tmp_header_title="$(cat $1 | head -n 1 | sed 's/^\# Benchmark name/benchmark/')"
+            tmp_result="$(cat $1 | grep '^[^\#]')"
+
+            printf "%s\n%s" "$tmp_header_title" "$tmp_result"                \
+              | awk -F',' -v me="$tmp_run_time_mean_i"                       \
+                        -v md="$tmp_run_time_median_i"                       \
+                        -v th="$tmp_run_time_throughput_i"                   \
+                  '{printf "%-57s %-13s %-15s %-20s\n", $1, $me, $md, $th }' \
+              | tee -a $3   # Print to summary file
+        }
+
+        # run sycl bench step
+        run() {
+            TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
+            mkdir "./sycl-bench/build/bench-$TIMESTAMP/"
+            tmp_summary_file="./sycl-bench/build/bench-$TIMESTAMP/summary.txt" 
+
+            for file in ./sycl-bench/build/bin/*; do
+                # TODO -size should not be always 256, caution
+                tmp_bench_output="./sycl-bench/build/bench-$TIMESTAMP/$(basename $file).csv"
+                tmp_bench_log="./sycl-bench/build/bench-$TIMESTAMP/$(basename $file).log"
+
+                tmp_err="0"
+                printf "\n### Results for $(basename $file) ###\n" | tee -a $tmp_summary_file
+                $file --output=$tmp_bench_output --no-verification --size=256 2> "$tmp_bench_log" || tmp_err=$?
+                print_bench_res $tmp_bench_output $tmp_err $tmp_summary_file
+                # Remove log if nothing logged
+                [ ! -s "$tmp_bench_log" ] && rm "$tmp_bench_log" || cat "$tmp_bench_log" | tee -a $tmp_summary_file
+            done
+
+            # Export timestamp for later use
+            echo TIMESTAMP=$TIMESTAMP >> $GITHUB_ENV
+        }
+
+        run
+    - name: Check results
+      if: steps.run-benchmarks.outcome == 'success'
+      run: |
+        cat "./sycl-bench/build/bench-$TIMESTAMP/summary.txt"
+    - name: Pack results
+      if: steps.run-benchmarks.outcome == 'success'
+      id: pack_results
+      run: |
+        ARCHIVE_PATH="./sycl-bench/build/bench-$TIMESTAMP.tar.gz"
+        tar -I gzip -cf "$ARCHIVE_PATH" -C "./sycl-bench/build/bench-$TIMESTAMP" .
+        echo ARCHIVE_PATH="$ARCHIVE_PATH" >> $GITHUB_OUTPUT
+    - name: Upload results
+      if: steps.run-benchmarks.outcome == 'success'
+      uses: actions/upload-artifact@v4
+      with:
+        name: sycl_benchmark_res_${{ env.TIMESTAMP }}
+        path: ${{ steps.pack_results.outputs.ARCHIVE_PATH }}
+        retention-days: 7

.github/workflows/sycl-linux-precommit.yml

@@ -69,67 +69,68 @@ jobs:
           else
             echo 'arc_tests="Matrix/"' >> "$GITHUB_OUTPUT"
           fi
-  test:
-    needs: [build, detect_changes, determine_arc_tests]
-    if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - name: AMD/HIP
-            runner: '["Linux", "amdgpu"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_build:latest-0300ac924620a51f76c4929794637b82790f12ab
-            image_options: -u 1001 --device=/dev/dri --device=/dev/kfd
-            target_devices: ext_oneapi_hip:gpu
-          - name: Intel
-            runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
-            image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
-            target_devices: ext_oneapi_level_zero:gpu;opencl:gpu;opencl:cpu
-            reset_gpu: true
-            install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}
-            extra_lit_opts: --param gpu-intel-gen12=True
-          - name: E2E tests on Intel Arc A-Series Graphics
-            runner: '["Linux", "arc"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
-            image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
-            target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
-            reset_gpu: true
-            install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}
-            extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True
-            env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }'
-          - name: E2E tests with dev igc on Intel Arc A-Series Graphics
-            runner: '["Linux", "arc"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }}
-            image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
-            target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
-            reset_gpu: true
-            install_drivers: >-
-              ${{ contains(needs.detect_changes.outputs.filters, 'drivers') ||
-              contains(needs.detect_changes.outputs.filters, 'devigccfg') }}
-            use_dev_igc: ${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }}
-            extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True
-            env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }'
+  # test:
+  #   needs: [build, detect_changes, determine_arc_tests]
+  #   if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       include:
+  #         # TODO UNDO THIS
+  #         # - name: AMD/HIP
+  #         #   runner: '["Linux", "amdgpu"]'
+  #         #   image: ghcr.io/intel/llvm/ubuntu2204_build:latest-0300ac924620a51f76c4929794637b82790f12ab
+  #         #   image_options: -u 1001 --device=/dev/dri --device=/dev/kfd
+  #         #   target_devices: ext_oneapi_hip:gpu
+  #         - name: Intel
+  #           runner: '["Linux", "gen12"]'
+  #           image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+  #           image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+  #           target_devices: ext_oneapi_level_zero:gpu;opencl:gpu;opencl:cpu
+  #           reset_gpu: true
+  #           install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}
+  #           extra_lit_opts: --param gpu-intel-gen12=True
+  #         # - name: E2E tests on Intel Arc A-Series Graphics
+  #         #   runner: '["Linux", "arc"]'
+  #         #   image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+  #         #   image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+  #         #   target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
+  #         #   reset_gpu: true
+  #         #   install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}
+  #         #   extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True
+  #         #   env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }'
+  #         # - name: E2E tests with dev igc on Intel Arc A-Series Graphics
+  #         #   runner: '["Linux", "arc"]'
+  #         #   image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }}
+  #         #   image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+  #         #   target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
+  #         #   reset_gpu: true
+  #         #   install_drivers: >-
+  #         #     ${{ contains(needs.detect_changes.outputs.filters, 'drivers') ||
+  #         #     contains(needs.detect_changes.outputs.filters, 'devigccfg') }}
+  #         #   use_dev_igc: ${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }}
+  #         #   extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True
+  #         #   env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }'
 
-    uses: ./.github/workflows/sycl-linux-run-tests.yml
-    with:
-      name: ${{ matrix.name }}
-      runner: ${{ matrix. runner }}
-      image: ${{ matrix.image }}
-      image_options: ${{ matrix.image_options }}
-      target_devices: ${{ matrix.target_devices }}
-      reset_gpu: ${{ matrix.reset_gpu }}
-      install_drivers: ${{ matrix.install_drivers }}
-      use_dev_igc: ${{ matrix.use_dev_igc }}
-      extra_lit_opts: ${{ matrix.extra_lit_opts }}
-      env: ${{ matrix.env || '{}' }}
+  #   uses: ./.github/workflows/sycl-linux-run-tests.yml
+  #   with:
+  #     name: ${{ matrix.name }}
+  #     runner: ${{ matrix. runner }}
+  #     image: ${{ matrix.image }}
+  #     image_options: ${{ matrix.image_options }}
+  #     target_devices: ${{ matrix.target_devices }}
+  #     reset_gpu: ${{ matrix.reset_gpu }}
+  #     install_drivers: ${{ matrix.install_drivers }}
+  #     use_dev_igc: ${{ matrix.use_dev_igc }}
+  #     extra_lit_opts: ${{ matrix.extra_lit_opts }}
+  #     env: ${{ matrix.env || '{}' }}
 
-      ref: ${{ github.sha }}
-      merge_ref: ''
+  #     ref: ${{ github.sha }}
+  #     merge_ref: ''
 
-      sycl_toolchain_artifact: sycl_linux_default
-      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
-      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+  #     sycl_toolchain_artifact: sycl_linux_default
+  #     sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
+  #     sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
 
 
   test-perf:
@@ -181,3 +182,38 @@ jobs:
       sycl_toolchain_artifact: sycl_linux_default
       sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
       sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+
+  sycl-benchmark:
+    needs: [build] #, test]
+    if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # - name: AMD/HIP
+          #   runner: '["Linux", "amdgpu"]'
+          #   image: ghcr.io/intel/llvm/ubuntu2204_build:latest-0300ac924620a51f76c4929794637b82790f12ab
+          #   image_options: -u 1001 --device=/dev/dri --device=/dev/kfd
+          #   target_devices: ext_oneapi_hip:gpu
+          - name: Intel
+            runner: '["Linux", "gen12"]'
+            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+          # - name: E2E tests on Intel Arc A-Series Graphics
+          #   runner: '["Linux", "arc"]'
+          #   image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+          #   image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+          #   target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
+          # - name: E2E tests with dev igc on Intel Arc A-Series Graphics
+          #   runner: '["Linux", "arc"]'
+          #   image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }}
+          #   image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+          #   target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
+    uses: ./.github/workflows/sycl-benchmark.yml
+    with:
+      runner: ${{ matrix.runner }}
+      image: ${{ matrix.image }}
+      image_options: ${{ matrix.image_options }}
+      sycl_toolchain_artifact: sycl_linux_default
+      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
+      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}

.github/workflows/sycl-linux-run-tests.yml

@@ -339,7 +339,8 @@ jobs:
         ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }}
       # This job takes ~100min usually. But sometimes some test isn't
       # responding, so the job reaches the 360min limit. Setting a lower one.
-      timeout-minutes: 150
+      # TODO UNDO THIS
+      timeout-minutes: 1
       # By-default GitHub actions execute the "run" shell script with -e option,
       # so the execution terminates if any command returns a non-zero status.
       # Since we're using a loop to run all test-binaries separately, some test