benchmark.yaml

name: benchmark

on:
  pull_request:
    paths-ignore:
      - "website/**"

jobs:
  criterion:
    name: criterion
    runs-on: ubuntu-22.04
    steps:
      - name: checkout ${{ github.base_ref }}
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
        with:
          ref: ${{ github.base_ref }}

      - name: setup environment
        uses: ./.github/actions/setup

      - name: measure benchmarks on ${{ github.base_ref }}
        run: cargo bench
        working-directory: libs/benches

      - name: checkout PR
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
        with:
          clean: false

      - name: setup environment
        uses: ./.github/actions/setup

      - name: measure benchmarks on PR
        run: cargo bench
        working-directory: libs/benches

      - name: upload benchmark report
        uses: actions/upload-artifact@v4
        with:
          name: Criterion Report
          path: target/criterion/

  k6:
    name: k6
    env:
      K6_VERSION: 0.48.0
    runs-on: ubuntu-22.04
    steps:
      - name: adjust os
        run: ulimit -n 30000

      - name: checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

      - name: setup environment
        uses: ./.github/actions/setup

      - name: Setup K6
        run: |
          sudo gpg -k
          sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69
          echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
          sudo apt-get update
          sudo apt-get install k6=${{ env.K6_VERSION }}

      - name: "build release binary"
        run: cargo build --bin conductor --release

      - uses: JarvusInnovations/background-action@v1
        name: run server in background
        with:
          run: cargo run --release
          wait-on: http-get://127.0.0.1:4000/_health
          tail: true
          wait-for: 10m
          log-output-if: failure
          log-output: true
          working-directory: tests/test-server

      - uses: JarvusInnovations/background-action@v1
        name: run gateway in background
        with:
          run: cargo run --release --bin conductor -- ./benchmark/gw.yaml
          wait-on: http-get://127.0.0.1:9000/_health
          tail: true
          wait-for: 10m
          log-output-if: failure
          log-output: true

      - name: run k6
        run: |
          k6 --summary-trend-stats "min,avg,med,max,p(95),p(99)" -e GITHUB_PR=$GITHUB_PR -e GITHUB_SHA=$GITHUB_SHA -e GITHUB_TOKEN=$GITHUB_TOKEN run k6.js
        working-directory: benchmark
        env:
          GITHUB_PR: ${{ github.event.number }}
          GITHUB_SHA: ${{ github.sha }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}