Skip to content

Automation Benchmark Test #233

Automation Benchmark Test

Automation Benchmark Test #233

name: Automation Benchmark Test
on:
workflow_dispatch:
inputs:
chainlinkVersion:
description: Chainlink image version to use
required: true
type: string
default: 2.5.0
chainlinkImage:
description: Chainlink image repo to use
required: true
type: string
default: public.ecr.aws/chainlink/chainlink
network:
description: Network to run tests on
required: true
type: choice
options:
- SIMULATED
- SIMULATED_NONDEV
- GOERLI
- ARBITRUM_GOERLI
- OPTIMISM_GOERLI
- MUMBAI
- SEPOLIA
- BASE_GOERLI
- ARBITRUM_SEPOLIA
TestInputs:
description: TestInputs
required: false
type: string
wsURL:
description: WS URL for the network (Skip for Simulated)
required: false
type: string
httpURL:
description: HTTP URL for the network (Skip for Simulated)
required: false
type: string
slackMemberID:
description: Notifies test results (Not your @)
required: true
default: U02Q14G80TY
type: string
fundingPrivateKey:
description: Private funding key (Skip for Simulated)
required: false
type: string
jobs:
automation_benchmark:
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
name: ${{ inputs.network }} Automation Benchmark Test
runs-on: ubuntu-latest
env:
SELECTED_NETWORKS: ${{ inputs.network }}
SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
SLACK_CHANNEL: C03KJ5S7KEK
TEST_INPUTS: ${{ inputs.TestInputs }}
CHAINLINK_ENV_USER: ${{ github.actor }}
REF_NAME: ${{ github.head_ref || github.ref_name }}
steps:
- name: Setup Push Tag
shell: bash
run: |
echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
echo "\`${{ inputs.chainlinkVersion }}\`" >>$GITHUB_STEP_SUMMARY
echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
- name: Add mask
run: |
EVM_URLS=$(jq -r '.inputs.wsURL' $GITHUB_EVENT_PATH)
EVM_HTTP_URLS=$(jq -r '.inputs.httpURL' $GITHUB_EVENT_PATH)
EVM_KEYS=$(jq -r '.inputs.fundingPrivateKey' $GITHUB_EVENT_PATH)
SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
echo ::add-mask::$EVM_URLS
echo ::add-mask::$EVM_HTTP_URLS
echo ::add-mask::$EVM_KEYS
echo ::add-mask::$SLACK_USER
echo EVM_URLS=$EVM_URLS >> $GITHUB_ENV
echo EVM_HTTP_URLS=$EVM_HTTP_URLS >> $GITHUB_ENV
echo EVM_KEYS=$EVM_KEYS >> $GITHUB_ENV
echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
while IFS=',' read -ra EVM_URLS_2; do
for i in "${EVM_URLS_2[@]}"; do
echo ::add-mask::$i
done
done <<< "$EVM_URLS"
while IFS=',' read -ra EVM_HTTP_URLS_2; do
for i in "${EVM_HTTP_URLS_2[@]}"; do
echo ::add-mask::$i
done
done <<< "$EVM_HTTP_URLS"
- name: Checkout the repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ env.REF_NAME }}
- name: Build Test Image
uses: ./.github/actions/build-test-image
with:
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@7d541cbbca52d45b8a718257af86d9cf49774d1f # v2.2.15
env:
DETACH_RUNNER: true
TEST_SUITE: benchmark
TEST_ARGS: -test.timeout 720h
ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
with:
test_command_to_run: cd integration-tests && go test -timeout 30m -v -run ^TestAutomationBenchmark$ ./benchmark -count=1
test_download_vendor_packages_command: make gomod
cl_repo: ${{ inputs.chainlinkImage }}
cl_image_tag: ${{ inputs.chainlinkVersion }}
token: ${{ secrets.GITHUB_TOKEN }}
should_cleanup: false
go_mod_path: ./integration-tests/go.mod
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
- name: Collect Metrics
if: always()
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
with:
basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: ${{ inputs.network }} Automation Benchmark Test
test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
continue-on-error: true