Performance Regression Test - Score Director #27
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # - Runs entirely on a single machine. | |
| # - The baseline is established first, then the branch under test is measured. | |
| # - Both runs fail if the benchmark error is over predefined thresholds. | |
| # - Then, if both are below thresholds and neither failed, those results must be directly comparable. | |
| # - Therefore, if the difference between the two is over the threshold, then the branch is considered to have regressed. | |
| name: Performance Regression Test - Score Director | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| jdk: | |
| description: 'JDK version' | |
| default: '21' | |
| required: true | |
| baseline: | |
| description: 'Timefold Solver release' | |
| default: '1.14.0' | |
| required: true | |
| branch: | |
| description: 'Branch to benchmark (needs to use 999-SNAPSHOT)' | |
| default: 'main' | |
| required: true | |
| branch_owner: | |
| description: 'User owning the branch' | |
| default: 'TimefoldAI' | |
| required: true | |
| async_profiler_version: | |
| description: 'async-profiler version' | |
| default: '3.0' | |
| required: true | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false # Jobs fail if the benchmark error is over predefined thresholds; other benchmarks continue. | |
| matrix: | |
| example: [cloud_balancing, conference_scheduling, curriculum_course, examination, machine_reassignment, meeting_scheduling, nurse_rostering, patient_admission_scheduling, task_assigning, traveling_tournament, tsp, vehicle_routing] | |
| env: | |
| MVN_USERNAME: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_USERNAME }}' | |
| MVN_PASSWORD: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_TOKEN }}' | |
| steps: | |
| - name: Phase 0 - Checkout timefold-solver-benchmarks | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: TimefoldAI/timefold-solver-benchmarks | |
| path: ./timefold-solver-benchmarks | |
| - name: Phase 0 - Setup JDK and Maven | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ github.event.inputs.jdk }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| server-id: 'timefold-solver-enterprise' | |
| server-username: 'MVN_USERNAME' | |
| server-password: 'MVN_PASSWORD' | |
| - name: Phase 0 - Setup Async Profiler | |
| working-directory: ./timefold-solver-benchmarks | |
| run: | | |
| export FILENAME=async-profiler-${{ github.event.inputs.async_profiler_version }}-linux-x64.tar.gz | |
| wget https://github.com/async-profiler/async-profiler/releases/download/v${{ github.event.inputs.async_profiler_version }}/$FILENAME | |
| tar -xzf $FILENAME | |
| ls -l | |
| - name: Phase 0 - Prepare the benchmarks | |
| working-directory: ./timefold-solver-benchmarks | |
| shell: bash | |
| run: | | |
| echo "forks=15" > scoredirector-benchmark.properties | |
| echo "score_director_type=cs" >> scoredirector-benchmark.properties | |
| echo "example=${{ matrix.example }}" >> scoredirector-benchmark.properties | |
| cat scoredirector-benchmark.properties | |
| chmod +x run-scoredirector.sh | |
| - name: Phase 1 - Compile the benchmarks | |
| working-directory: ./timefold-solver-benchmarks | |
| shell: bash | |
| run: mvn clean install -B -Dquickly -Dversion.ai.timefold.solver=${{ github.event.inputs.baseline }} -Dversion.tools.provider="${{ github.event.inputs.async_profiler_version }}" | |
| - name: Phase 1 - Run the baseline configuration | |
| working-directory: ./timefold-solver-benchmarks | |
| shell: bash | |
| run: | | |
| export RUN_ID="${{github.workflow.inputs.baseline}}" | |
| ./run-scoredirector.sh | |
| - name: Phase 2 - Checkout timefold-solver | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.branch_owner }}/timefold-solver | |
| ref: ${{ github.event.inputs.branch }} | |
| path: ./timefold-solver | |
| - name: Phase 2 - Quickly build timefold-solver | |
| working-directory: ./timefold-solver | |
| shell: bash | |
| run: mvn -B -Dquickly clean install | |
| - name: Phase 2 - Checkout timefold-solver-enterprise | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: TimefoldAI/timefold-solver-enterprise | |
| ref: ${{ github.event.inputs.branch }} | |
| token: ${{ secrets.JRELEASER_GITHUB_TOKEN }} # Safe; only used to clone the repo and not stored in the fork. | |
| path: ./timefold-solver-enterprise | |
| - name: Phase 2 - Quickly build timefold-solver-enterprise | |
| working-directory: ./timefold-solver-enterprise | |
| shell: bash | |
| run: mvn -B -Dquickly clean install | |
| - name: Phase 2 - Compile the benchmarks | |
| working-directory: ./timefold-solver-benchmarks | |
| shell: bash | |
| run: mvn clean install -B -Dquickly -Dversion.tools.provider="${{ github.event.inputs.async_profiler_version }}" | |
| - name: Phase 2 - Run the new configuration | |
| working-directory: ./timefold-solver-benchmarks | |
| shell: bash | |
| run: | | |
| export RUN_ID="${{github.workflow.inputs.branch}}" | |
| ./run-scoredirector.sh | |
| - name: Phase 3 - Archive benchmark data | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.example }}-${{ github.event.inputs.baseline }}_vs_${{ github.event.inputs.branch }} | |
| path: | | |
| ./timefold-solver-benchmarks/results | |
| - name: Phase 3 - Publish benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: scoredirector-${{ matrix.example }} | |
| tool: 'jmh' | |
| output-file-path: ./timefold-solver-benchmarks/results/${{ github.event.inputs.branch }}/results.json | |
| github-token: ${{ secrets.BENCHMARK_PUBLISH_TOKEN }} | |
| auto-push: true | |
| alert-threshold: '105%' | |
| comment-on-alert: true | |
| fail-on-alert: true | |
| alert-comment-cc-users: '@triceo' |