diff --git a/.github/workflows/sycl-benchmark.yml b/.github/workflows/sycl-benchmark.yml new file mode 100644 index 0000000000000..9c9f56531e700 --- /dev/null +++ b/.github/workflows/sycl-benchmark.yml @@ -0,0 +1,215 @@ +name: Run benchmarks (using sycl-bench) + +on: + workflow_call: + inputs: + runner: + type: string + required: True + image: + type: string + required: True + image_options: + type: string + required: True + + sycl_toolchain_artifact: + type: string + default: '' + required: False + sycl_toolchain_archive: + type: string + default: '' + required: False + sycl_toolchain_decompress_command: + type: string + default: '' + required: False + + workflow_dispatch: + inputs: + runner: + type: choice + options: + - '["Linux", "gen12"]' + - '["amdgpu"]' + - '["Linux", "arc"]' + - '["cts-cpu"]' + image: + description: | + Use option ending with ":build" for AMDGPU, ":latest" for the rest. + type: choice + options: + - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:latest' + - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build' + image_options: + description: | + Use option with "--device=/dev/kfd" for AMDGPU, without it for the rest. + type: choice + options: + - '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN' + - '-u 1001 --device=/dev/dri --device=/dev/kfd --privileged --cap-add SYS_ADMIN' + + sycl_toolchain_artifact: + type: string + default: '' + required: False + sycl_toolchain_archive: + type: string + default: '' + required: False + sycl_toolchain_decompress_command: + type: string + default: '' + required: False + + +permissions: + contents: read + +jobs: + run: + name: Run benchmarks using sycl-bench + runs-on: ${{ fromJSON(inputs.runner || '["Linux", "gen12"]') }} + container: + image: ${{ inputs.image }} + options: ${{ inputs.image_options }} + steps: + - name: Download SYCL toolchain + if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.sycl_toolchain_artifact }} + - name: Debug prints [workflow_run] + if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' + run: | + pwd + ls + - name: Download SYCL toolchain [workflow_run] + if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' + uses: actions/github-script@v7 + with: + script: | + const name = '${{ inputs.sycl_toolchain_artifact }}' + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == name + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/` + name + '.zip', Buffer.from(download.data)); + - name: Unzip artifact [workflow_run] + if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' + run: | + pwd + ls + unzip ${{ inputs.sycl_toolchain_artifact }}.zip + rm ${{ inputs.sycl_toolchain_artifact }}.zip + - name: Extract/Setup SYCL toolchain + if: inputs.sycl_toolchain_artifact != '' + run: | + mkdir toolchain + tar -I '${{ inputs.sycl_toolchain_decompress_command }}' -xf ${{ inputs.sycl_toolchain_archive }} -C toolchain + rm -f ${{ inputs.sycl_toolchain_archive }} + echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV + echo SYCL_ARTIFACT_PATH="$PWD/toolchain/" >> $GITHUB_ENV + echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV + - uses: actions/checkout@v4 + with: + repository: 'ianayl/sycl-bench' + clean: false # Prevents actions/checkout@v4 from deleting the extracted artifact + path: './sycl-bench/' + - name: Build sycl-bench + run: | + cd ./sycl-bench + cmake -DSYCL_IMPL=dpcpp -DCMAKE_CXX_COMPILER=$SYCL_ARTIFACT_PATH/bin/clang++ -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=./bin -S . -B ./build && + cmake --build ./build + cd - + echo LD_LIBRARY_PATH=$SYCL_ARTIFACT_PATH/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV + - name: Run sycl-bench tests + id: run-benchmarks + run: | + # Determine the index of a column in a CSV given its title + # Usage: get_csv_col_index + get_csv_col_index() { + tmp_csv_col_i="$(cat "$1" | head -n 1 | grep -o "^.*$2," | grep -o ',' | wc -l)" + } + + # Usage: print_bench_res + print_bench_res() { + if [ ! -s $1 ]; then + printf "NO OUTPUT! (Status $2)\n" | tee -a $3 + return # Do not proceed if file is empty + fi + + get_csv_col_index $1 run-time-mean + tmp_run_time_mean_i=$tmp_csv_col_i + get_csv_col_index $1 run-time-median + tmp_run_time_median_i=$tmp_csv_col_i + get_csv_col_index $1 run-time-throughput + tmp_run_time_throughput_i=$tmp_csv_col_i + + # `sycl-bench` output seems to like inserting the header multiple times. + # Here we cache the header to make sure it prints only once: + tmp_header_title="$(cat $1 | head -n 1 | sed 's/^\# Benchmark name/benchmark/')" + tmp_result="$(cat $1 | grep '^[^\#]')" + + printf "%s\n%s" "$tmp_header_title" "$tmp_result" \ + | awk -F',' -v me="$tmp_run_time_mean_i" \ + -v md="$tmp_run_time_median_i" \ + -v th="$tmp_run_time_throughput_i" \ + '{printf "%-57s %-13s %-15s %-20s\n", $1, $me, $md, $th }' \ + | tee -a $3 # Print to summary file + } + + # run sycl bench step + run() { + TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" + mkdir "./sycl-bench/build/bench-$TIMESTAMP/" + tmp_summary_file="./sycl-bench/build/bench-$TIMESTAMP/summary.txt" + + for file in ./sycl-bench/build/bin/*; do + # TODO -size should not be always 256, caution + tmp_bench_output="./sycl-bench/build/bench-$TIMESTAMP/$(basename $file).csv" + tmp_bench_log="./sycl-bench/build/bench-$TIMESTAMP/$(basename $file).log" + + tmp_err="0" + printf "\n### Results for $(basename $file) ###\n" | tee -a $tmp_summary_file + $file --output=$tmp_bench_output --no-verification --size=256 2> "$tmp_bench_log" || tmp_err=$? + print_bench_res $tmp_bench_output $tmp_err $tmp_summary_file + # Remove log if nothing logged + [ ! -s "$tmp_bench_log" ] && rm "$tmp_bench_log" || cat "$tmp_bench_log" | tee -a $tmp_summary_file + done + + # Export timestamp for later use + echo TIMESTAMP=$TIMESTAMP >> $GITHUB_ENV + } + + run + - name: Check results + if: steps.run-benchmarks.outcome == 'success' + run: | + cat "./sycl-bench/build/bench-$TIMESTAMP/summary.txt" + - name: Pack results + if: steps.run-benchmarks.outcome == 'success' + id: pack_results + run: | + ARCHIVE_PATH="./sycl-bench/build/bench-$TIMESTAMP.tar.gz" + tar -I gzip -cf "$ARCHIVE_PATH" -C "./sycl-bench/build/bench-$TIMESTAMP" . + echo ARCHIVE_PATH="$ARCHIVE_PATH" >> $GITHUB_OUTPUT + - name: Upload results + if: steps.run-benchmarks.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: sycl_benchmark_res_${{ env.TIMESTAMP }} + path: ${{ steps.pack_results.outputs.ARCHIVE_PATH }} + retention-days: 7 diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 424ade23f1b08..256ef0032a4ed 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -76,11 +76,11 @@ jobs: fail-fast: false matrix: include: - - name: NVIDIA/CUDA - runner: '["Linux", "cuda"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest - image_options: -u 1001 --gpus all --cap-add SYS_ADMIN - target_devices: ext_oneapi_cuda:gpu + # - name: NVIDIA/CUDA + # runner: '["Linux", "cuda"]' + # image: ghcr.io/intel/llvm/ubuntu2204_build:latest + # image_options: -u 1001 --gpus all --cap-add SYS_ADMIN + # target_devices: ext_oneapi_cuda:gpu - name: Intel runner: '["Linux", "gen12"]' image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest @@ -89,27 +89,27 @@ jobs: reset_intel_gpu: true install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} extra_lit_opts: --param gpu-intel-gen12=True - - name: E2E tests on Intel Arc A-Series Graphics - runner: '["Linux", "arc"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest - image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN - target_devices: level_zero:gpu;opencl:gpu - reset_intel_gpu: true - install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} - extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True - env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' - - name: E2E tests with dev igc on Intel Arc A-Series Graphics - runner: '["Linux", "arc"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }} - image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN - target_devices: level_zero:gpu;opencl:gpu - reset_intel_gpu: true - install_drivers: >- - ${{ contains(needs.detect_changes.outputs.filters, 'drivers') || - contains(needs.detect_changes.outputs.filters, 'devigccfg') }} - use_dev_igc: ${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }} - extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True - env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' + # - name: E2E tests on Intel Arc A-Series Graphics + # runner: '["Linux", "arc"]' + # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest + # image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + # target_devices: level_zero:gpu;opencl:gpu + # reset_intel_gpu: true + # install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} + # extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True + # env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' + # - name: E2E tests with dev igc on Intel Arc A-Series Graphics + # runner: '["Linux", "arc"]' + # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }} + # image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + # target_devices: level_zero:gpu;opencl:gpu + # reset_intel_gpu: true + # install_drivers: >- + # ${{ contains(needs.detect_changes.outputs.filters, 'drivers') || + # contains(needs.detect_changes.outputs.filters, 'devigccfg') }} + # use_dev_igc: ${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }} + # extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True + # env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' uses: ./.github/workflows/sycl-linux-run-tests.yml with: @@ -124,12 +124,12 @@ jobs: extra_lit_opts: ${{ matrix.extra_lit_opts }} env: ${{ matrix.env || '{}' }} - ref: ${{ github.sha }} - merge_ref: '' + # ref: ${{ github.sha }} + # merge_ref: '' - sycl_toolchain_artifact: sycl_linux_default - sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} - sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} + # sycl_toolchain_artifact: sycl_linux_default + # sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} + # sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} test-perf: @@ -181,3 +181,38 @@ jobs: sycl_toolchain_artifact: sycl_linux_default sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} + + sycl-benchmark: + needs: [build] #, test] + if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} + strategy: + fail-fast: false + matrix: + include: + # - name: AMD/HIP + # runner: '["Linux", "amdgpu"]' + # image: ghcr.io/intel/llvm/ubuntu2204_build:latest-0300ac924620a51f76c4929794637b82790f12ab + # image_options: -u 1001 --device=/dev/dri --device=/dev/kfd + # target_devices: ext_oneapi_hip:gpu + - name: Intel + runner: '["Linux", "gen12"]' + image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + # - name: E2E tests on Intel Arc A-Series Graphics + # runner: '["Linux", "arc"]' + # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest + # image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + # target_devices: ext_oneapi_level_zero:gpu;opencl:gpu + # - name: E2E tests with dev igc on Intel Arc A-Series Graphics + # runner: '["Linux", "arc"]' + # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }} + # image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + # target_devices: ext_oneapi_level_zero:gpu;opencl:gpu + uses: ./.github/workflows/sycl-benchmark.yml + with: + runner: ${{ matrix.runner }} + image: ${{ matrix.image }} + image_options: ${{ matrix.image_options }} + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} \ No newline at end of file diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 1f3cbd5f156b1..ecfb5da5e97e0 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -348,7 +348,8 @@ jobs: ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} # This job takes ~100min usually. But sometimes some test isn't # responding, so the job reaches the 360min limit. Setting a lower one. - timeout-minutes: 150 + # TODO UNDO THIS + timeout-minutes: 1 # By-default GitHub actions execute the "run" shell script with -e option, # so the execution terminates if any command returns a non-zero status. # Since we're using a loop to run all test-binaries separately, some test