automation-benchmark-tests.yml

name: Automation Benchmark Test
on:
  workflow_dispatch:
    inputs:
      chainlinkVersion:
        description: Chainlink image version to use
        required: true
        type: string
        default: 2.5.0
      chainlinkImage:
        description: Chainlink image repo to use
        required: true
        type: string
        default: public.ecr.aws/chainlink/chainlink
      network:
        description: Network to run tests on
        required: true
        type: choice
        options:
          - SIMULATED
          - SIMULATED_NONDEV
          - GOERLI
          - ARBITRUM_GOERLI
          - OPTIMISM_GOERLI
          - MUMBAI
          - SEPOLIA
          - BASE_GOERLI
          - ARBITRUM_SEPOLIA
      TestInputs:
        description: TestInputs
        required: false
        type: string
      wsURL:
        description: WS URL for the network (Skip for Simulated)
        required: false
        type: string
      httpURL:
        description: HTTP URL for the network (Skip for Simulated)
        required: false
        type: string
      slackMemberID:
        description: Notifies test results (Not your @)
        required: true
        default: U02Q14G80TY
        type: string
      fundingPrivateKey:
        description: Private funding key (Skip for Simulated)
        required: false
        type: string

jobs:
  automation_benchmark:
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    name: ${{ inputs.network }} Automation Benchmark Test
    runs-on: ubuntu20.04-16cores-64GB
    env:
      SELECTED_NETWORKS: ${{ inputs.network }}
      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
      SLACK_CHANNEL: C03KJ5S7KEK
      TEST_INPUTS: ${{ inputs.TestInputs }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      REF_NAME: ${{ github.head_ref || github.ref_name }}
    steps:
      - name: Setup Push Tag
        shell: bash
        run: |
          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
          echo "\`${{ inputs.chainlinkVersion }}\`" >>$GITHUB_STEP_SUMMARY
          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY

      - name: Add mask
        run: |
          EVM_URLS=$(jq -r '.inputs.wsURL' $GITHUB_EVENT_PATH)
          EVM_HTTP_URLS=$(jq -r '.inputs.httpURL' $GITHUB_EVENT_PATH)
          EVM_KEYS=$(jq -r '.inputs.fundingPrivateKey' $GITHUB_EVENT_PATH)
          SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
          echo ::add-mask::$EVM_URLS
          echo ::add-mask::$EVM_HTTP_URLS
          echo ::add-mask::$EVM_KEYS
          echo ::add-mask::$SLACK_USER
          echo EVM_URLS=$EVM_URLS >> $GITHUB_ENV
          echo EVM_HTTP_URLS=$EVM_HTTP_URLS >> $GITHUB_ENV
          echo EVM_KEYS=$EVM_KEYS >> $GITHUB_ENV
          echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
          while IFS=',' read -ra EVM_URLS_2; do
          for i in "${EVM_URLS_2[@]}"; do
              echo ::add-mask::$i
            done
          done <<< "$EVM_URLS"
          while IFS=',' read -ra EVM_HTTP_URLS_2; do
            for i in "${EVM_HTTP_URLS_2[@]}"; do
              echo ::add-mask::$i
            done
          done <<< "$EVM_HTTP_URLS"
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ env.REF_NAME }}
      - name: Build Test Image
        uses: ./.github/actions/build-test-image
        with:
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
      - name: Run Tests
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@7d541cbbca52d45b8a718257af86d9cf49774d1f # v2.2.15
        env:
          DETACH_RUNNER: true
          TEST_SUITE: benchmark
          TEST_ARGS: -test.timeout 720h
          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
          INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
        with:
          test_command_to_run: cd integration-tests && go test -timeout 30m -v -run ^TestAutomationBenchmark$ ./benchmark -count=1
          test_download_vendor_packages_command: make gomod
          cl_repo: ${{ inputs.chainlinkImage }}
          cl_image_tag: ${{ inputs.chainlinkVersion }}
          token: ${{ secrets.GITHUB_TOKEN }}
          should_cleanup: false
          go_mod_path: ./integration-tests/go.mod
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
        with:
          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
          this-job-name: ${{ inputs.network }} Automation Benchmark Test
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true