Skip to content

Benchmark Large

Benchmark Large #272

# Copyright 2023 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# Executes large benchmarks that are expected to take multple hours.
name: Benchmark Large
on:
schedule:
# Scheduled to run at 09:00 UTC and 21:00 UTC.
- cron: '0 09,21 * * *'
workflow_dispatch:
inputs:
shard-count:
description: |
A device-name to integer mapping as a comma separated list.
Allows to assign a distinct shard count for each target device.
The reserved keyword `default` assigns a shard count to all target devices
that are not explicitly listed.
# Please keep this default value in sync with the jobs.build_e2e_test_artifacts.with.shard-count field below
default: a2-highgpu-1g=1,c2-standard-16=2,default=1
type: string
concurrency:
# A PR number if a pull request and otherwise the commit hash. This cancels
# queued and in-progress runs for the same PR (presubmit) or commit
# (postsubmit). The workflow name is prepended to avoid conflicts between
# different workflows.
group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
cancel-in-progress: true
env:
# This needs to be in env instead of the outputs of setup because it contains
# the run attempt and we want that to be the current attempt, not whatever
# attempt the setup step last ran in.
GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }}
# Jobs are organized into groups and topologically sorted by dependencies
jobs:
setup:
uses: ./.github/workflows/setup.yml
build_all:
needs: setup
if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_all')
uses: ./.github/workflows/build_all.yml
with:
runner-group: ${{ needs.setup.outputs.runner-group }}
runner-env: ${{ needs.setup.outputs.runner-env }}
write-caches: ${{ needs.setup.outputs.write-caches }}
build_benchmark_tools:
needs: [setup, build_all]
if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_benchmark_tools')
uses: ./.github/workflows/build_benchmark_tools.yml
with:
runner-group: ${{ needs.setup.outputs.runner-group }}
runner-env: ${{ needs.setup.outputs.runner-env }}
build-dir: ${{ needs.build_all.outputs.build-dir }}
build-dir-archive: ${{ needs.build_all.outputs.build-dir-archive }}
build-dir-gcs-artifact: ${{ needs.build_all.outputs.build-dir-gcs-artifact }}
build_e2e_test_artifacts:
needs: [setup, build_all]
if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_e2e_test_artifacts')
uses: ./.github/workflows/build_e2e_test_artifacts.yml
with:
runner-group: ${{ needs.setup.outputs.runner-group }}
runner-env: ${{ needs.setup.outputs.runner-env }}
build-dir: ${{ needs.build_all.outputs.build-dir }}
build-dir-archive: ${{ needs.build_all.outputs.build-dir-archive }}
build-dir-gcs-artifact: ${{ needs.build_all.outputs.build-dir-gcs-artifact }}
benchmark-presets: cuda-large,comp-stats-large,x86_64-large
build-default-benchmark-suites: 0
# Please keep the shard count default value in sync with on.workflow_dispatch.shard-count.default
shard-count: ${{ inputs && inputs.shard_count || 'a2-highgpu-1g=1,c2-standard-16=2,default=1' }}
compilation_benchmarks:
needs: [setup, build_e2e_test_artifacts]
if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'compilation_benchmarks')
uses: ./.github/workflows/benchmark_compilation.yml
with:
runner-group: ${{ needs.setup.outputs.runner-group }}
runner-env: ${{ needs.setup.outputs.runner-env }}
e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }}
e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }}
e2e-test-artifacts-build-log: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log }}
e2e-test-artifacts-build-log-gcs-artifact: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log-gcs-artifact }}
execution_benchmarks:
needs: [setup, build_benchmark_tools, build_e2e_test_artifacts]
if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'execution_benchmarks')
uses: ./.github/workflows/benchmark_execution.yml
with:
# env.GCS_DIR is also duplicated in this workflow. See the note there on
# why this is.
runner-group: ${{ needs.setup.outputs.runner-group }}
runner-env: ${{ needs.setup.outputs.runner-env }}
e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }}
e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }}
benchmark-tools-gcs-artifact-dir: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }}
summary:
# Even if you have an explicit if condition, you still need to override
# GitHub's default behavior of not running if any dependencies failed.
if: always()
runs-on: ubuntu-20.04
needs:
- setup
- build_all
- build_benchmark_tools
- build_e2e_test_artifacts
- compilation_benchmarks
- execution_benchmarks
steps:
- name: Getting failed jobs
id: failed_jobs
run: |
echo '${{ toJson(needs) }}'
FAILED_JOBS="$(echo '${{ toJson(needs) }}' \
| jq --raw-output \
'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \
)"
echo "failed-jobs=${FAILED_JOBS}" >> $GITHUB_OUTPUT
if [[ "${FAILED_JOBS}" != "" ]]; then
echo "The following jobs failed: ${FAILED_JOBS}"
exit 1
fi
- name: Posting to Discord
uses: sarisia/actions-status-discord@61114b793b460ee85fe38ad3fccc78c7ead38d55 # v1.11.1
if: failure() && github.ref_name == 'main'
with:
webhook: ${{ secrets.DISCORD_WEBHOOK }}
description: "The following jobs failed: ${{ steps.failed_jobs.outputs.failed-jobs }}"