-
Notifications
You must be signed in to change notification settings - Fork 1.7k
137 lines (135 loc) · 5.26 KB
/
automation-benchmark-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
name: Automation Benchmark Test
on:
workflow_dispatch:
inputs:
chainlinkVersion:
description: Chainlink image version to use
required: true
type: string
default: 2.5.0
chainlinkImage:
description: Chainlink image repo to use
required: true
type: string
default: public.ecr.aws/chainlink/chainlink
network:
description: Network to run tests on
required: true
type: choice
options:
- SIMULATED
- SIMULATED_NONDEV
- GOERLI
- ARBITRUM_GOERLI
- OPTIMISM_GOERLI
- MUMBAI
- SEPOLIA
TestInputs:
description: TestInputs
required: false
type: string
wsURL:
description: WS URL for the network (Skip for Simulated)
required: false
type: string
httpURL:
description: HTTP URL for the network (Skip for Simulated)
required: false
type: string
slackMemberID:
description: Notifies test results (Not your @)
required: true
default: U02Q14G80TY
type: string
fundingPrivateKey:
description: Private funding key (Skip for Simulated)
required: false
type: string
jobs:
automation_benchmark:
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
name: ${{ inputs.network }} Automation Benchmark Test
runs-on: ubuntu-latest
env:
SELECTED_NETWORKS: ${{ inputs.network }}
SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
SLACK_CHANNEL: C03KJ5S7KEK
TEST_INPUTS: ${{ inputs.TestInputs }}
CHAINLINK_ENV_USER: ${{ github.actor }}
REF_NAME: ${{ github.head_ref || github.ref_name }}
steps:
- name: Setup Push Tag
shell: bash
run: |
echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
echo "\`${{ inputs.chainlinkVersion }}\`" >>$GITHUB_STEP_SUMMARY
echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
- name: Add mask
run: |
EVM_URLS=$(jq -r '.inputs.wsURL' $GITHUB_EVENT_PATH)
EVM_HTTP_URLS=$(jq -r '.inputs.httpURL' $GITHUB_EVENT_PATH)
EVM_KEYS=$(jq -r '.inputs.fundingPrivateKey' $GITHUB_EVENT_PATH)
SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
echo ::add-mask::$EVM_URLS
echo ::add-mask::$EVM_HTTP_URLS
echo ::add-mask::$EVM_KEYS
echo ::add-mask::$SLACK_USER
echo EVM_URLS=$EVM_URLS >> $GITHUB_ENV
echo EVM_HTTP_URLS=$EVM_HTTP_URLS >> $GITHUB_ENV
echo EVM_KEYS=$EVM_KEYS >> $GITHUB_ENV
echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
while IFS=',' read -ra EVM_URLS_2; do
for i in "${EVM_URLS_2[@]}"; do
echo ::add-mask::$i
done
done <<< "$EVM_URLS"
while IFS=',' read -ra EVM_HTTP_URLS_2; do
for i in "${EVM_HTTP_URLS_2[@]}"; do
echo ::add-mask::$i
done
done <<< "$EVM_HTTP_URLS"
- name: Checkout the repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ env.REF_NAME }}
- name: Build Test Image
uses: ./.github/actions/build-test-image
with:
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
env:
DETACH_RUNNER: true
TEST_SUITE: benchmark
TEST_ARGS: -test.timeout 720h
ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
with:
test_command_to_run: cd integration-tests && go test -timeout 30m -v -run ^TestAutomationBenchmark$ ./benchmark -count=1
test_download_vendor_packages_command: make gomod
cl_repo: ${{ inputs.chainlinkImage }}
cl_image_tag: ${{ inputs.chainlinkVersion }}
token: ${{ secrets.GITHUB_TOKEN }}
should_cleanup: false
go_mod_path: ./integration-tests/go.mod
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
- name: Collect Metrics
if: always()
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
with:
basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: ${{ inputs.network }} Automation Benchmark Test
test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
continue-on-error: true