[DO NOT MERGE] Test PR for new benchmarking workflow

.github/workflows/sycl-benchmark.yml

@@ -0,0 +1,215 @@
+name: Run benchmarks (using sycl-bench)
+
+on:
+ workflow_call:
+ inputs:
+ runner:
+ type: string
+ required: True
+ image:
+ type: string
+ required: True
+ image_options:
+ type: string
+ required: True
+
+ sycl_toolchain_artifact:
+ type: string
+ default: ''
+ required: False
+ sycl_toolchain_archive:
+ type: string
+ default: ''
+ required: False
+ sycl_toolchain_decompress_command:
+ type: string
+ default: ''
+ required: False
+
+ workflow_dispatch:
+ inputs:
+ runner:
+ type: choice
+ options:
+ - '["Linux", "gen12"]'
+ - '["amdgpu"]'
+ - '["Linux", "arc"]'
+ - '["cts-cpu"]'
+ image:
+ description: |
+ Use option ending with ":build" for AMDGPU, ":latest" for the rest.
+ type: choice
+ options:
+ - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:latest'
+ - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build'
+ image_options:
+ description: |
+ Use option with "--device=/dev/kfd" for AMDGPU, without it for the rest.
+ type: choice
+ options:
+ - '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN'
+ - '-u 1001 --device=/dev/dri --device=/dev/kfd --privileged --cap-add SYS_ADMIN'
+
+ sycl_toolchain_artifact:
+ type: string
+ default: ''
+ required: False
+ sycl_toolchain_archive:
+ type: string
+ default: ''
+ required: False
+ sycl_toolchain_decompress_command:
+ type: string
+ default: ''
+ required: False
+
+
+permissions:
+ contents: read
+
+jobs:
+ run:
+ name: Run benchmarks using sycl-bench
+ runs-on: ${{ fromJSON(inputs.runner || '["Linux", "gen12"]') }}
+ container:
+ image: ${{ inputs.image }}
+ options: ${{ inputs.image_options }}
+ steps:
+ - name: Download SYCL toolchain
+ if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run'
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ inputs.sycl_toolchain_artifact }}
+ - name: Debug prints [workflow_run]
+ if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
+ run: |
+ pwd
+ ls
+ - name: Download SYCL toolchain [workflow_run]
+ if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
+ uses: actions/github-script@v7
+ with:
+ script: |
+ const name = '${{ inputs.sycl_toolchain_artifact }}'
+ let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ run_id: context.payload.workflow_run.id,
+ });
+ let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+ return artifact.name == name
+ })[0];
+ let download = await github.rest.actions.downloadArtifact({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ artifact_id: matchArtifact.id,
+ archive_format: 'zip',
+ });
+ let fs = require('fs');
+ fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/` + name + '.zip', Buffer.from(download.data));
+ - name: Unzip artifact [workflow_run]
+ if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
+ run: |
+ pwd
+ ls
+ unzip ${{ inputs.sycl_toolchain_artifact }}.zip
+ rm ${{ inputs.sycl_toolchain_artifact }}.zip
+ - name: Extract/Setup SYCL toolchain
+ if: inputs.sycl_toolchain_artifact != ''
+ run: |
+ mkdir toolchain
+ tar -I '${{ inputs.sycl_toolchain_decompress_command }}' -xf ${{ inputs.sycl_toolchain_archive }} -C toolchain
+ rm -f ${{ inputs.sycl_toolchain_archive }}
+ echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV
+ echo SYCL_ARTIFACT_PATH="$PWD/toolchain/" >> $GITHUB_ENV
+ echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV
+ - uses: actions/checkout@v4
+ with:
+ repository: 'ianayl/sycl-bench'
+ clean: false # Prevents actions/checkout@v4 from deleting the extracted artifact
+ path: './sycl-bench/'
+ - name: Build sycl-bench
+ run: |
+ cd ./sycl-bench
+ cmake -DSYCL_IMPL=dpcpp -DCMAKE_CXX_COMPILER=$SYCL_ARTIFACT_PATH/bin/clang++ -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=./bin -S . -B ./build &&
+ cmake --build ./build
+ cd -
+ echo LD_LIBRARY_PATH=$SYCL_ARTIFACT_PATH/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV
+ - name: Run sycl-bench tests
+ id: run-benchmarks
+ run: |
+ # Determine the index of a column in a CSV given its title
+ # Usage: get_csv_col_index <benchmark output .csv file> <column name>
+ get_csv_col_index() {
+ tmp_csv_col_i="$(cat "$1" | head -n 1 | grep -o "^.*$2," | grep -o ',' | wc -l)"
+ }
+
+ # Usage: print_bench_res <benchmark output .csv file> <benchmark status code> <summary file>
+ print_bench_res() {
+ if [ ! -s $1 ]; then
+ printf "NO OUTPUT! (Status $2)\n" | tee -a $3
+ return # Do not proceed if file is empty
+ fi
+
+ get_csv_col_index $1 run-time-mean
+ tmp_run_time_mean_i=$tmp_csv_col_i
+ get_csv_col_index $1 run-time-median
+ tmp_run_time_median_i=$tmp_csv_col_i
+ get_csv_col_index $1 run-time-throughput
+ tmp_run_time_throughput_i=$tmp_csv_col_i
+
+ # `sycl-bench` output seems to like inserting the header multiple times.
+ # Here we cache the header to make sure it prints only once:
+ tmp_header_title="$(cat $1 | head -n 1 | sed 's/^\# Benchmark name/benchmark/')"
+ tmp_result="$(cat $1 | grep '^[^\#]')"
+
+ printf "%s\n%s" "$tmp_header_title" "$tmp_result" \
+ | awk -F',' -v me="$tmp_run_time_mean_i" \
+ -v md="$tmp_run_time_median_i" \
+ -v th="$tmp_run_time_throughput_i" \
+ '{printf "%-57s %-13s %-15s %-20s\n", $1, $me, $md, $th }' \
+ | tee -a $3 # Print to summary file
+ }
+
+ # run sycl bench step
+ run() {
+ TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
+ mkdir "./sycl-bench/build/bench-$TIMESTAMP/"
+ tmp_summary_file="./sycl-bench/build/bench-$TIMESTAMP/summary.txt" 
+
+ for file in ./sycl-bench/build/bin/*; do
+ # TODO -size should not be always 256, caution
+ tmp_bench_output="./sycl-bench/build/bench-$TIMESTAMP/$(basename $file).csv"
+ tmp_bench_log="./sycl-bench/build/bench-$TIMESTAMP/$(basename $file).log"
+
+ tmp_err="0"
+ printf "\n### Results for $(basename $file) ###\n" | tee -a $tmp_summary_file
+ $file --output=$tmp_bench_output --no-verification --size=256 2> "$tmp_bench_log" || tmp_err=$?
+ print_bench_res $tmp_bench_output $tmp_err $tmp_summary_file
+ # Remove log if nothing logged
+ [ ! -s "$tmp_bench_log" ] && rm "$tmp_bench_log" || cat "$tmp_bench_log" | tee -a $tmp_summary_file
+ done
+
+ # Export timestamp for later use
+ echo TIMESTAMP=$TIMESTAMP >> $GITHUB_ENV
+ }
+
+ run
+ - name: Check results
+ if: steps.run-benchmarks.outcome == 'success'
+ run: |
+ cat "./sycl-bench/build/bench-$TIMESTAMP/summary.txt"
+ - name: Pack results
+ if: steps.run-benchmarks.outcome == 'success'
+ id: pack_results
+ run: |
+ ARCHIVE_PATH="./sycl-bench/build/bench-$TIMESTAMP.tar.gz"
+ tar -I gzip -cf "$ARCHIVE_PATH" -C "./sycl-bench/build/bench-$TIMESTAMP" .
+ echo ARCHIVE_PATH="$ARCHIVE_PATH" >> $GITHUB_OUTPUT
+ - name: Upload results
+ if: steps.run-benchmarks.outcome == 'success'
+ uses: actions/upload-artifact@v4
+ with:
+ name: sycl_benchmark_res_${{ env.TIMESTAMP }}
+ path: ${{ steps.pack_results.outputs.ARCHIVE_PATH }}
+ retention-days: 7

.github/workflows/sycl-linux-precommit.yml

@@ -76,11 +76,11 @@ jobs:
  fail-fast: false
  matrix:
  include:
- - name: NVIDIA/CUDA
- runner: '["Linux", "cuda"]'
- image: ghcr.io/intel/llvm/ubuntu2204_build:latest
- image_options: -u 1001 --gpus all --cap-add SYS_ADMIN
- target_devices: ext_oneapi_cuda:gpu
+ # - name: NVIDIA/CUDA
+ #  runner: '["Linux", "cuda"]'
+ #  image: ghcr.io/intel/llvm/ubuntu2204_build:latest
+ #  image_options: -u 1001 --gpus all --cap-add SYS_ADMIN
+ #  target_devices: ext_oneapi_cuda:gpu
  - name: Intel
  runner: '["Linux", "gen12"]'
  image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
@@ -89,27 +89,27 @@ jobs:
  reset_intel_gpu: true
  install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}
  extra_lit_opts: --param gpu-intel-gen12=True
- - name: E2E tests on Intel Arc A-Series Graphics
- runner: '["Linux", "arc"]'
- image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
- image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
- target_devices: level_zero:gpu;opencl:gpu
- reset_intel_gpu: true
- install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}
- extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True
- env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }'
- - name: E2E tests with dev igc on Intel Arc A-Series Graphics
- runner: '["Linux", "arc"]'
- image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }}
- image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
- target_devices: level_zero:gpu;opencl:gpu
- reset_intel_gpu: true
- install_drivers: >-
- ${{ contains(needs.detect_changes.outputs.filters, 'drivers') ||
- contains(needs.detect_changes.outputs.filters, 'devigccfg') }}
- use_dev_igc: ${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }}
- extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True
- env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }'
+ # - name: E2E tests on Intel Arc A-Series Graphics
+ #  runner: '["Linux", "arc"]'
+ #  image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+ #  image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+ #  target_devices: level_zero:gpu;opencl:gpu
+ #  reset_intel_gpu: true
+ #  install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}
+ #  extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True
+ #  env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }'
+ # - name: E2E tests with dev igc on Intel Arc A-Series Graphics
+ #  runner: '["Linux", "arc"]'
+ #  image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }}
+ #  image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+ #  target_devices: level_zero:gpu;opencl:gpu
+ #  reset_intel_gpu: true
+ #  install_drivers: >-
+ #  ${{ contains(needs.detect_changes.outputs.filters, 'drivers') ||
+ #  contains(needs.detect_changes.outputs.filters, 'devigccfg') }}
+ #  use_dev_igc: ${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }}
+ #  extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True
+ #  env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }'
 
  uses: ./.github/workflows/sycl-linux-run-tests.yml
  with:
@@ -124,12 +124,12 @@ jobs:
  extra_lit_opts: ${{ matrix.extra_lit_opts }}
  env: ${{ matrix.env || '{}' }}
 
- ref: ${{ github.sha }}
- merge_ref: ''
+ #  ref: ${{ github.sha }}
+ #  merge_ref: ''
 
- sycl_toolchain_artifact: sycl_linux_default
- sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
- sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+ #  sycl_toolchain_artifact: sycl_linux_default
+ #  sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
+ #  sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
 
 
  test-perf:
@@ -181,3 +181,38 @@ jobs:
  sycl_toolchain_artifact: sycl_linux_default
  sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
  sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+
+ sycl-benchmark:
+ needs: [build] #, test]
+ if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ # - name: AMD/HIP
+ # runner: '["Linux", "amdgpu"]'
+ # image: ghcr.io/intel/llvm/ubuntu2204_build:latest-0300ac924620a51f76c4929794637b82790f12ab
+ # image_options: -u 1001 --device=/dev/dri --device=/dev/kfd
+ # target_devices: ext_oneapi_hip:gpu
+ - name: Intel
+ runner: '["Linux", "gen12"]'
+ image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+ image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+ # - name: E2E tests on Intel Arc A-Series Graphics
+ # runner: '["Linux", "arc"]'
+ # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+ # image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+ # target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
+ # - name: E2E tests with dev igc on Intel Arc A-Series Graphics
+ # runner: '["Linux", "arc"]'
+ # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }}
+ # image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+ # target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
+ uses: ./.github/workflows/sycl-benchmark.yml
+ with:
+ runner: ${{ matrix.runner }}
+ image: ${{ matrix.image }}
+ image_options: ${{ matrix.image_options }}
+ sycl_toolchain_artifact: sycl_linux_default
+ sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
+ sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}

.github/workflows/sycl-linux-run-tests.yml

@@ -348,7 +348,8 @@ jobs:
  ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }}
  # This job takes ~100min usually. But sometimes some test isn't
  # responding, so the job reaches the 360min limit. Setting a lower one.
- timeout-minutes: 150
+ # TODO UNDO THIS
+ timeout-minutes: 1
  # By-default GitHub actions execute the "run" shell script with -e option,
  # so the execution terminates if any command returns a non-zero status.
  # Since we're using a loop to run all test-binaries separately, some test