improve benchmark scripts and add a nightly job

This patch: - Adds a benchmark suite abstraction and moves all existing benchmarks into it. This makes individual benchmark types self-contained and allows us to skip over benchmarks that don't have their dependencies met. - Makes sycl and ur optional, and they are no longer positional arguments. - Creates a benchmark history that stores benchmark runs. This then enables us to do comparisions not just with the latest result but also against e.g., a historical average. - adds a nightly job to store baseline results. This is all in preparation for adding an HTML output that will generate a static HTML with historical benchmark data.
oneapi-src · Nov 4, 2024 · 9e72ff2 · 9e72ff2
1 parent fa8cc8e
commit 9e72ff2
Show file tree

Hide file tree

Showing 19 changed files with 690 additions and 436 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -0,0 +1,77 @@
+name: Compute Benchmarks
+
+on:
+  workflow_dispatch:
+    inputs:
+      str_name:
+        description: Formatted adapter name
+        type: choice
+        required: true
+        default: 'level_zero'
+        options:
+          - level_zero
+          - level_zero_v2
+      unit:
+        description: Test unit (cpu/gpu)
+        type: choice
+        required: true
+        default: 'gpu'
+        options:
+          - cpu
+          - gpu
+      pr_no:
+        description: PR number (if 0, it'll run on the main)
+        type: number
+        required: true
+      bench_script_params:
+        description: Parameters passed to script executing benchmark
+        type: string
+        required: false
+        default: ''
+      sycl_config_params:
+        description: Extra params for SYCL configuration
+        type: string
+        required: false
+        default: ''
+      sycl_repo:
+        description: 'Compiler repo'
+        type: string
+        required: true
+        default: 'intel/llvm'
+      sycl_commit:
+        description: 'Compiler commit'
+        type: string
+        required: false
+        default: ''
+
+  schedule:
+    - cron: '0 0 * * *'  # Runs at midnight UTC every day
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  manual:
+    name: Manual Compute Benchmarks
+    uses: ./.github/workflows/compute-benchmarks-reusable.yml
+    with:
+      str_name: ${{ inputs.str_name }}
+      unit: ${{ inputs.unit }}
+      pr_no: ${{ inputs.pr_no }}
+      bench_script_params: ${{ inputs.bench_script_params }}
+      sycl_config_params: ${{ inputs.sycl_config_params }}
+      sycl_repo: ${{ inputs.sycl_repo }}
+      sycl_commit: ${{ inputs.sycl_commit }}
+
+  nightly:
+    name: Nightly Compute Benchmarks
+    uses: ./.github/workflows/compute-benchmarks-reusable.yml
+    with:
+      str_name: 'level_zero'
+      unit: 'gpu'
+      pr_no: 0
+      bench_script_params: '--save baseline'
+      sycl_config_params: ''
+      sycl_repo: 'intel/llvm'
+      sycl_commit: ''
diff --git a/.github/workflows/benchmarks_compute.yml → .github/workflows/benchmarks_reusable.yml b/.github/workflows/benchmarks_compute.yml → .github/workflows/benchmarks_reusable.yml
@@ -1,49 +1,32 @@
-name: Compute Benchmarks
+name: Benchmarks Reusable
 
 on:
-  # Can be triggered via manual "dispatch" (from workflow view in GitHub Actions tab)
-  workflow_dispatch:
-    # acceptable input for adapter-specific runs
+  workflow_call:
     inputs:
       str_name:
-        description: Formatted adapter name
-        type: choice
         required: true
-        default: 'level_zero'
-        options:
-          - level_zero
-          - level_zero_v2
+        type: string
       unit:
-        description: Test unit (cpu/gpu)
-        type: choice
         required: true
-        default: 'gpu'
-        options:
-          - cpu
-          - gpu
+        type: string
       pr_no:
-        description: PR number (if 0, it'll run on the main)
-        type: number
         required: true
+        type: number
       bench_script_params:
-        description: Parameters passed to script executing benchmark
-        type: string
         required: false
+        type: string
         default: ''
       sycl_config_params:
-        description: Extra params for SYCL configuration
-        type: string
         required: false
+        type: string
         default: ''
       sycl_repo:
-        description: 'Compiler repo'
-        type: string
         required: true
+        type: string
         default: 'intel/llvm'
       sycl_commit:
-        description: 'Compiler commit'
-        type: string
         required: false
+        type: string
         default: ''
 
 permissions:
@@ -56,27 +39,25 @@ jobs:
     strategy:
       matrix:
         adapter: [
-          {str_name: "${{inputs.str_name}}",
-          sycl_config: "${{inputs.sycl_config_params}}",
-          unit: "${{inputs.unit}}"
+          {str_name: "${{ inputs.str_name }}",
+          sycl_config: "${{ inputs.sycl_config_params }}",
+          unit: "${{ inputs.unit }}"
           }
         ]
         build_type: [Release]
         compiler: [{c: clang, cxx: clang++}]
 
-    runs-on: "${{inputs.str_name}}_PERF"
+    runs-on: "${{ inputs.str_name }}_PERF"
 
     steps:
-    # Workspace on self-hosted runners is not cleaned automatically.
-    # We have to delete the files created outside of using actions.
     - name: Cleanup self-hosted workspace
       if: always()
       run: |
         ls -la ./
         rm -rf ./* || true
 
     - name: Add comment to PR
-      uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+      uses: actions/github-script@v7
       if: ${{ always() && inputs.pr_no != 0 }}
       with:
         script: |
@@ -94,14 +75,13 @@ jobs:
           })
 
     - name: Checkout UR
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      uses: actions/checkout@v4
       with:
         path: ur-repo
 
     - name: Install pip packages
       run: pip install -r ${{github.workspace}}/ur-repo/third_party/requirements.txt
 
-    # We need to fetch special ref for proper PR's merge commit. Note, this ref may be absent if the PR is already merged.
     - name: Fetch PR's merge commit
       if: ${{ inputs.pr_no != 0 }}
       working-directory: ${{github.workspace}}/ur-repo
@@ -113,9 +93,9 @@ jobs:
         git rev-parse origin/pr/${PR_NO}/merge
 
     - name: Checkout SYCL
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      uses: actions/checkout@v4
       with:
-        repository: ${{inputs.sycl_repo}}
+        repository: ${{ inputs.sycl_repo }}
         ref: refs/heads/sycl
         path: sycl-repo
         fetch-depth: 1
@@ -137,49 +117,49 @@ jobs:
     - name: Configure SYCL
       run: >
         python3 sycl-repo/buildbot/configure.py
-        -t ${{matrix.build_type}}
-        -o ${{github.workspace}}/sycl_build
+        -t ${{ matrix.build_type }}
+        -o ${{ github.workspace }}/sycl_build
         --cmake-gen "Ninja"
-        --ci-defaults ${{matrix.adapter.sycl_config}}
+        --ci-defaults ${{ matrix.adapter.sycl_config }}
         --cmake-opt="-DLLVM_INSTALL_UTILS=ON"
         --cmake-opt="-DSYCL_PI_TESTS=OFF"
         --cmake-opt="-DSYCL_UR_USE_FETCH_CONTENT=OFF"
-        --cmake-opt="-DSYCL_UR_SOURCE_DIR=${{github.workspace}}/ur-repo/"
+        --cmake-opt="-DSYCL_UR_SOURCE_DIR=${{ github.workspace }}/ur-repo/"
         --cmake-opt=-DCMAKE_C_COMPILER_LAUNCHER=ccache
         --cmake-opt=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 
     - name: Build SYCL
-      run: cmake --build ${{github.workspace}}/sycl_build -j
+      run: cmake --build ${{ github.workspace }}/sycl_build -j
 
     - name: Configure UR
       run: >
         cmake -DCMAKE_BUILD_TYPE=Release
-        -S${{github.workspace}}/ur-repo
-        -B${{github.workspace}}/ur_build
-        -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/ur_install
+        -S${{ github.workspace }}/ur-repo
+        -B${{ github.workspace }}/ur_build
+        -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/ur_install
         -DUR_BUILD_TESTS=OFF
         -DUR_BUILD_ADAPTER_L0=ON
         -DUR_BUILD_ADAPTER_L0_V2=ON
         -DUMF_DISABLE_HWLOC=ON
 
     - name: Build UR
-      run: cmake --build ${{github.workspace}}/ur_build -j $(nproc)
+      run: cmake --build ${{ github.workspace }}/ur_build -j $(nproc)
 
     - name: Install UR
-      run: cmake --install ${{github.workspace}}/ur_build
+      run: cmake --install ${{ github.workspace }}/ur_build
 
     - name: Run benchmarks
       id: benchmarks
       run: >
         numactl -N 0 ${{ github.workspace }}/ur-repo/scripts/benchmarks/main.py 
         ~/bench_workdir 
-        ${{github.workspace}}/sycl_build
-        ${{github.workspace}}/ur_install
-        ${{ matrix.adapter.str_name }}
+        --sycl ${{ github.workspace }}/sycl_build
+        --ur ${{ github.workspace }}/ur_install
+        --adapter ${{ matrix.adapter.str_name }}
         ${{ inputs.bench_script_params }}
 
     - name: Add comment to PR
-      uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+      uses: actions/github-script@v7
       if: ${{ always() && inputs.pr_no != 0 }}
       with:
         script: |

diff --git a/scripts/benchmarks/benches/SobelFilter.py b/scripts/benchmarks/benches/SobelFilter.py
diff --git a/scripts/benchmarks/benches/base.py b/scripts/benchmarks/benches/base.py
@@ -20,16 +20,18 @@ def __init__(self, directory):
     def get_adapter_full_path():
         for libs_dir_name in ['lib', 'lib64']:
             adapter_path = os.path.join(
-                options.ur_dir, libs_dir_name, f"libur_adapter_{options.ur_adapter_name}.so")
+                options.ur, libs_dir_name, f"libur_adapter_{options.ur_adapter}.so")
             if os.path.isfile(adapter_path):
                 return adapter_path
         assert False, \
             f"could not find adapter file {adapter_path} (and in similar lib paths)"
 
     def run_bench(self, command, env_vars):
         env_vars_with_forced_adapter = env_vars.copy()
-        env_vars_with_forced_adapter.update(
-            {'UR_ADAPTERS_FORCE_LOAD': Benchmark.get_adapter_full_path()})
+        if options.ur is not None:
+            env_vars_with_forced_adapter.update(
+                {'UR_ADAPTERS_FORCE_LOAD': Benchmark.get_adapter_full_path()})
+
         return run(
             command=command,
             env_vars=env_vars_with_forced_adapter,
@@ -76,3 +78,10 @@ def run(self, env_vars) -> list[Result]:
 
     def teardown(self):
         raise NotImplementedError()
+
+class Suite:
+    def benchmarks(self) -> list[Benchmark]:
+        raise NotImplementedError()
+
+    def setup(self):
+        return
diff --git a/scripts/benchmarks/benches/bitcracker.py b/scripts/benchmarks/benches/bitcracker.py