Skip to content

.github/workflows/ccip-chaos-tests.yml #1

.github/workflows/ccip-chaos-tests.yml

.github/workflows/ccip-chaos-tests.yml #1

name: CCIP Chaos Tests
on:
workflow_run:
workflows: [ CCIP Load Test ]
types: [ completed ]
branches: [ develop ]
workflow_dispatch:
# Only run 1 of this workflow at a time per PR
concurrency:
group: chaos-ccip-tests-chainlink-${{ github.ref }}
cancel-in-progress: true
env:
# TODO: TT-1470 - Update image names as we solidify new realease strategy
CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }}
MOD_CACHE_VERSION: 1
jobs:
build-chainlink:
environment: integration
permissions:
id-token: write
contents: read
name: Build Chainlink Image
runs-on: ubuntu20.04-16cores-64GB
steps:
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Check if image exists
id: check-image
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
with:
repository: chainlink
tag: ${{ github.sha }}
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
- name: Build Image
if: steps.check-image.outputs.exists == 'false'
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
env:
GH_TOKEN: ${{ github.token }}
with:
cl_repo: smartcontractkit/chainlink
cl_ref: ${{ github.sha }}
push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ github.sha }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
- name: Collect Metrics
if: always()
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
with:
id: ccip-chaos-tests-build-chainlink-image
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
this-job-name: Build Chainlink Image
continue-on-error: true
build-test-image:
environment: integration
permissions:
id-token: write
contents: read
name: Build Test Image
runs-on: ubuntu20.04-16cores-64GB
steps:
- name: Collect Metrics
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
with:
id: ccip-chaos-tests-build-test-image
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
this-job-name: Build Test Image
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Build Test Image
uses: ./.github/actions/build-test-image
with:
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
ccip-chaos-tests:
environment: integration
permissions:
issues: read
checks: write
pull-requests: write
id-token: write
contents: read
name: CCIP Chaos Tests
runs-on: ubuntu-latest
needs: [ build-chainlink, build-test-image ]
env:
TEST_SUITE: chaos
TEST_ARGS: -test.timeout 30m
CHAINLINK_COMMIT_SHA: ${{ github.sha }}
CHAINLINK_ENV_USER: ${{ github.actor }}
TEST_TRIGGERED_BY: ccip-cron-chaos-eth
TEST_LOG_LEVEL: debug
DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
GH_TOKEN: ${{ github.token }}
steps:
- name: Collect Metrics
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
with:
id: ccip-chaos-tests
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
this-job-name: CCIP Chaos Tests
test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Prepare Base64 TOML override for CCIP secrets
uses: ./.github/actions/setup-create-base64-config-ccip
with:
runId: ${{ github.run_id }}
testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
chainlinkImage: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
chainlinkVersion: ${{ github.sha }}
lokiEndpoint: ${{ secrets.LOKI_URL }}
lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
grafanaUrl: ${{ vars.GRAFANA_URL }}
grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
- name: Run Chaos Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
with:
test_command_to_run: cd ./integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 -run 'TestChaosCCIP' ./chaos 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
test_download_vendor_packages_command: make gomod
cl_repo: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
cl_image_tag: ${{ github.sha }}
artifacts_location: ./integration-tests/chaos/logs
publish_check_name: CCIP Chaos Test Results
publish_report_paths: ./tests-chaos-report.xml
triggered_by: ${{ env.TEST_TRIGGERED_BY }}
token: ${{ secrets.GITHUB_TOKEN }}
go_mod_path: ./integration-tests/go.mod
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
CGO_ENABLED: "1"
aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
cache_restore_only: "true"
## Notify in slack if the job fails
- name: Notify Slack
if: failure() && github.event_name != 'workflow_dispatch'
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
env:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
with:
channel-id: "#ccip-testing"
slack-message: ":x: :mild-panic-intensifies: CCIP chaos tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}"
## Run Cleanup if the job succeeds
- name: cleanup
if: always()
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
with:
triggered_by: ${{ env.TEST_TRIGGERED_BY }}
ccip-chaos-with-load-tests:
environment: integration
permissions:
issues: read
checks: write
pull-requests: write
id-token: write
contents: read
name: CCIP Load With Chaos Tests
if: false # Disabled until CCIP-2555 is resolved
runs-on: ubuntu-latest
needs: [ build-chainlink, build-test-image ]
env:
TEST_SUITE: load
TEST_ARGS: -test.timeout 1h
CHAINLINK_COMMIT_SHA: ${{ github.sha }}
CHAINLINK_ENV_USER: ${{ github.actor }}
TEST_TRIGGERED_BY: ccip-cron-chaos-and-load-eth
TEST_LOG_LEVEL: debug
DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
GH_TOKEN: ${{ github.token }}
steps:
- name: Collect Metrics
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
with:
id: ccip-chaos-tests-with-load-test
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
this-job-name: CCIP load with chaos test
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Prepare Base64 TOML override for CCIP secrests
uses: ./.github/actions/setup-create-base64-config-ccip
with:
runId: ${{ github.run_id }}
testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
chainlinkImage: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
chainlinkVersion: ${{ github.sha }}
lokiEndpoint: ${{ secrets.LOKI_URL }}
lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
grafanaUrl: ${{ vars.GRAFANA_URL }}
grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests"
- name: Run Load With Chaos Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
with:
test_command_to_run: cd ./integration-tests/ccip-tests && go test -timeout 2h -count=1 -json -test.parallel 4 -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' ./load 2>&1 | tee /tmp/gotest.log | gotestfmt
test_download_vendor_packages_command: make gomod
cl_repo: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
cl_image_tag: ${{ github.sha }}
artifacts_location: ./integration-tests/load/logs
publish_check_name: CCIP Chaos With Load Test Results
publish_report_paths: ./tests-chaos-with-load-report.xml
triggered_by: ${{ env.TEST_TRIGGERED_BY }}
token: ${{ secrets.GITHUB_TOKEN }}
go_mod_path: ./integration-tests/go.mod
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
CGO_ENABLED: "1"
aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
cache_restore_only: "true"
## Notify in slack if the job fails
- name: Notify Slack
if: failure() && github.event_name != 'workflow_dispatch'
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
env:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
with:
channel-id: "#ccip-testing"
slack-message: ":x: :mild-panic-intensifies: CCIP chaos with load tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}"
## Run Cleanup if the job succeeds
- name: cleanup
if: always()
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
with:
triggered_by: ${{ env.TEST_TRIGGERED_BY }}