diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 3f5c531cfd2..a3f64e48530 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -164,6 +164,17 @@ runner-test-matrix: E2E_TEST_CHAINLINK_VERSION: '{{ env.DEFAULT_CHAINLINK_PLUGINS_VERSION }}' # This is the chainlink version that has the plugins ENABLE_OTEL_TRACES: true + - id: chaos/ocr_chaos_test.go + path: integration-tests/chaos/ocr_chaos_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + workflows: + - Automation On Demand Tests + - E2E Chaos Tests + test_cmd: cd integration-tests/chaos && DETACH_RUNNER=false go test -test.run "^TestOCRChaos$" -v -test.parallel=10 -timeout 60m -count=1 -json + test_env_vars: + TEST_SUITE: chaos + # END: OCR tests # START: Automation tests @@ -531,7 +542,7 @@ runner-test-matrix: test_env_vars: TEST_SUITE: reorg workflows: - - Run Automation On Demand Tests (TEST WORKFLOW) + - Automation On Demand Tests test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_0 -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-automation-on-demand-reorg @@ -542,7 +553,7 @@ runner-test-matrix: test_env_vars: TEST_SUITE: reorg workflows: - - Run Automation On Demand Tests (TEST WORKFLOW) + - Automation On Demand Tests test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_1 -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-automation-on-demand-reorg @@ -553,7 +564,7 @@ runner-test-matrix: test_env_vars: TEST_SUITE: reorg workflows: - - Run Automation On Demand Tests (TEST WORKFLOW) + - Automation On Demand Tests test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_2 -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-automation-on-demand-reorg @@ -564,7 +575,7 @@ runner-test-matrix: test_env_vars: TEST_SUITE: reorg workflows: - - Run Automation On Demand Tests (TEST WORKFLOW) + - Automation On Demand Tests test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_3 -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-automation-on-demand-reorg @@ -573,7 +584,8 @@ runner-test-matrix: test_env_type: k8s-remote-runner runs_on: ubuntu-latest workflows: - - Run Automation On Demand Tests (TEST WORKFLOW) + - Automation On Demand Tests + - E2E Chaos Tests test_cmd: cd integration-tests/chaos && DETACH_RUNNER=false go test -v -test.run ^TestAutomationChaos$ -test.parallel=20 -timeout 60m -count=1 -json pyroscope_env: ci-automation-on-demand-chaos test_env_vars: @@ -964,4 +976,30 @@ runner-test-matrix: test_env_vars: E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + - id: ccip-tests/chaos/ccip_test.go + path: integration-tests/ccip-tests/chaos/ccip_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + workflows: + - E2E CCIP Chaos Tests + test_cmd: cd integration-tests/ccip-tests/chaos && DETACH_RUNNER=false go test ccip_test.go -v -test.parallel=11 -timeout 60m -count=1 -json + test_env_vars: + TEST_SUITE: chaos + TEST_TRIGGERED_BY: ccip-cron-chaos-eth + TEST_LOG_LEVEL: debug + + - id: ccip-tests/load/ccip_test.go:^TestLoadCCIPStableWithPodChaosDiffCommitAndExec + path: integration-tests/ccip-tests/load/ccip_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + workflows: + # Disabled until CCIP-2555 is resolved + # - E2E CCIP Chaos Tests + test_cmd: cd integration-tests/ccip-tests/load && DETACH_RUNNER=false go test -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' -v -test.parallel=4 -timeout 120m -count=1 -json + test_env_vars: + TEST_SUITE: chaos + TEST_TRIGGERED_BY: ccip-cron-chaos-eth + TEST_LOG_LEVEL: debug + E2E_TEST_GRAFANA_DASHBOARD_URL: /d/6vjVx-1V8/ccip-long-running-tests + # END: CCIP tests \ No newline at end of file diff --git a/.github/workflows/ccip-chaos-tests.yml b/.github/workflows/ccip-chaos-tests.yml index 03f809b44c9..1d37551d9fe 100644 --- a/.github/workflows/ccip-chaos-tests.yml +++ b/.github/workflows/ccip-chaos-tests.yml @@ -11,248 +11,31 @@ concurrency: group: chaos-ccip-tests-chainlink-${{ github.ref }} cancel-in-progress: true -env: - # TODO: TT-1470 - Update image names as we solidify new realease strategy - CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }} - MOD_CACHE_VERSION: 1 - jobs: - build-chainlink: - environment: integration - permissions: - id-token: write - contents: read - name: Build Chainlink Image - runs-on: ubuntu20.04-16cores-64GB - steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Check if image exists - id: check-image - uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 - with: - repository: chainlink - tag: ${{ github.sha }} - AWS_REGION: ${{ secrets.QA_AWS_REGION }} - AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - name: Build Image - if: steps.check-image.outputs.exists == 'false' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 - env: - GH_TOKEN: ${{ github.token }} - with: - cl_repo: smartcontractkit/chainlink - cl_ref: ${{ github.sha }} - push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ github.sha }} - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 - with: - id: ccip-chaos-tests-build-chainlink-image - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Build Chainlink Image - continue-on-error: true - - build-test-image: - environment: integration - permissions: - id-token: write - contents: read - name: Build Test Image - runs-on: ubuntu20.04-16cores-64GB - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 - with: - id: ccip-chaos-tests-build-test-image - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Build Test Image - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Build Test Image - uses: ./.github/actions/build-test-image - with: - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - - ccip-chaos-tests: - environment: integration - permissions: - issues: read - checks: write - pull-requests: write - id-token: write - contents: read - name: CCIP Chaos Tests - runs-on: ubuntu-latest - needs: [ build-chainlink, build-test-image ] - env: - TEST_SUITE: chaos - TEST_ARGS: -test.timeout 30m - CHAINLINK_COMMIT_SHA: ${{ github.sha }} - CHAINLINK_ENV_USER: ${{ github.actor }} - TEST_TRIGGERED_BY: ccip-cron-chaos-eth - TEST_LOG_LEVEL: debug - DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable - GH_TOKEN: ${{ github.token }} - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 - with: - id: ccip-chaos-tests - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: CCIP Chaos Tests - test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Prepare Base64 TOML override for CCIP secrets - uses: ./.github/actions/setup-create-base64-config-ccip - id: setup_create_base64_config_ccip - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - chainlinkVersion: ${{ github.sha }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - - name: Run Chaos Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 - env: - BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} - TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} - with: - test_command_to_run: cd ./integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 -run 'TestChaosCCIP' ./chaos 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci - test_download_vendor_packages_command: make gomod - artifacts_location: ./integration-tests/chaos/logs - publish_check_name: CCIP Chaos Test Results - publish_report_paths: ./tests-chaos-report.xml - triggered_by: ${{ env.TEST_TRIGGERED_BY }} - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} - CGO_ENABLED: "1" - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} - DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} - DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} - DEFAULT_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} - DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - - ## Notify in slack if the job fails - - name: Notify Slack - if: failure() && github.event_name != 'workflow_dispatch' - uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 - env: - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} - with: - channel-id: "#ccip-testing" - slack-message: ":x: :mild-panic-intensifies: CCIP chaos tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" - ## Run Cleanup if the job succeeds - - name: cleanup - if: always() - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 - with: - triggered_by: ${{ env.TEST_TRIGGERED_BY }} - - ccip-chaos-with-load-tests: - environment: integration - permissions: - issues: read - checks: write - pull-requests: write - id-token: write - contents: read - name: CCIP Load With Chaos Tests - if: false # Disabled until CCIP-2555 is resolved - runs-on: ubuntu-latest - needs: [ build-chainlink, build-test-image ] - env: - TEST_SUITE: load - TEST_ARGS: -test.timeout 1h - CHAINLINK_COMMIT_SHA: ${{ github.sha }} - CHAINLINK_ENV_USER: ${{ github.actor }} - TEST_TRIGGERED_BY: ccip-cron-chaos-and-load-eth - TEST_LOG_LEVEL: debug - DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable - GH_TOKEN: ${{ github.token }} - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 - with: - id: ccip-chaos-tests-with-load-test - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: CCIP load with chaos test - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Prepare Base64 TOML override for CCIP secrests - uses: ./.github/actions/setup-create-base64-config-ccip - id: setup_create_base64_config_ccip - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - chainlinkVersion: ${{ github.sha }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - - name: Run Load With Chaos Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 - env: - BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} - TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} - with: - test_command_to_run: cd ./integration-tests/ccip-tests && go test -timeout 2h -count=1 -json -test.parallel 4 -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' ./load 2>&1 | tee /tmp/gotest.log | gotestfmt - test_download_vendor_packages_command: make gomod - artifacts_location: ./integration-tests/load/logs - publish_check_name: CCIP Chaos With Load Test Results - publish_report_paths: ./tests-chaos-with-load-report.xml - triggered_by: ${{ env.TEST_TRIGGERED_BY }} - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} - CGO_ENABLED: "1" - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} - DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} - DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} - DEFAULT_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} - DEFAULT_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests" - ## Notify in slack if the job fails - - name: Notify Slack - if: failure() && github.event_name != 'workflow_dispatch' - uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 - env: - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} - with: - channel-id: "#ccip-testing" - slack-message: ":x: :mild-panic-intensifies: CCIP chaos with load tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" - ## Run Cleanup if the job succeeds - - name: cleanup - if: always() - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 - with: - triggered_by: ${{ env.TEST_TRIGGERED_BY }} \ No newline at end of file + run-e2e-tests-workflow: + name: Run E2E Tests + uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml + with: + chainlink_version: ${{ github.sha }} + require_chainlink_image_versions_in_qa_ecr: ${{ github.sha }} + test_workflow: E2E CCIP Chaos Tests + test_log_level: debug + slack_notification_after_tests: on_failure + slack_notification_after_tests_channel_id: '#ccip-testing' + slack_notification_after_tests_name: CCIP Chaos E2E Tests + secrets: + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }} + GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} diff --git a/.github/workflows/integration-chaos-tests.yml b/.github/workflows/integration-chaos-tests.yml index 673614bf2c2..1f7e6f40fb1 100644 --- a/.github/workflows/integration-chaos-tests.yml +++ b/.github/workflows/integration-chaos-tests.yml @@ -7,148 +7,28 @@ on: - "*" workflow_dispatch: -env: - CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }} - TEST_SUITE: chaos - TEST_ARGS: -test.timeout 1h - CHAINLINK_COMMIT_SHA: ${{ github.sha }} - CHAINLINK_ENV_USER: ${{ github.actor }} - TEST_LOG_LEVEL: debug - jobs: - build-chainlink: - environment: integration - permissions: - id-token: write - contents: read - name: Build Chainlink Image - runs-on: ubuntu-latest - steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Check if image exists - id: check-image - uses: smartcontractkit/chainlink-github-actions/docker/image-exists@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 - with: - repository: chainlink - tag: ${{ github.sha }} - AWS_REGION: ${{ secrets.QA_AWS_REGION }} - AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - name: Build Image - if: steps.check-image.outputs.exists == 'false' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 - with: - cl_repo: smartcontractkit/chainlink - cl_ref: ${{ github.sha }} - push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ github.sha }} - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - name: Print Chainlink Image Built - id: push - run: | - echo "### chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY - echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: e2e-chaos-build-chainlink - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Build Chainlink Image - continue-on-error: true - - build-test-runner: - environment: integration - permissions: - id-token: write - contents: read - name: Build Test Runner Image - runs-on: ubuntu-latest - steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Build Test Image - uses: ./.github/actions/build-test-image - with: - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: e2e-chaos-build-test-image - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Build Test Runner Image - continue-on-error: true - - chaos-tests: - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - name: EVM Pods Chaos Tests - runs-on: ubuntu-latest - needs: [build-test-runner, build-chainlink] - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: e2e-chaos-tests - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: EVM Pods Chaos Tests - test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Prepare Base64 TOML config - env: - CHAINLINK_VERSION: ${{ github.sha }} - run: | - echo ::add-mask::$CHAINLINK_IMAGE - - cat << EOF > config.toml - [Network] - selected_networks=["SIMULATED"] - - [ChainlinkImage] - image="$CHAINLINK_IMAGE" - version="$CHAINLINK_VERSION" - EOF - - BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) - echo ::add-mask::$BASE64_CONFIG_OVERRIDE - echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV - - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27 - with: - test_command_to_run: cd integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 ./chaos 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - test_download_vendor_packages_command: cd ./integration-tests && go mod download - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ github.sha }} - artifacts_location: ./integration-tests/chaos/logs - publish_check_name: EVM Pods Chaos Test Results - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} - - name: Upload test log - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 - if: failure() - with: - name: Test Results Log - path: /tmp/gotest.log - retention-days: 7 + run-e2e-tests-workflow: + name: Run E2E Tests + uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml + with: + chainlink_version: ${{ github.sha }} + require_chainlink_image_versions_in_qa_ecr: ${{ github.sha }} + test_workflow: E2E Chaos Tests + test_log_level: debug + secrets: + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }} + GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml index 182e0f31d58..6644bba8d95 100644 --- a/.github/workflows/run-e2e-tests-reusable-workflow.yml +++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml @@ -52,10 +52,9 @@ on: required: false type: string slack_notification_after_tests: - description: 'Set to "true" to send a slack notification after the tests' + description: 'Set to "always" to always send a slack notification after the tests. Set "on_failure" to send a notification only on test failure' required: false - type: boolean - default: false + type: string slack_notification_after_tests_channel_id: description: 'Slack channel ID to send the notification to' required: false @@ -449,6 +448,7 @@ jobs: - name: Setup GAP for Grafana uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 id: setup-gap + timeout-minutes: 3 with: aws-region: ${{ secrets.AWS_REGION }} aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} @@ -499,7 +499,7 @@ jobs: E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary" - E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + E2E_TEST_GRAFANA_DASHBOARD_URL: ${{ matrix.tests.test_env_vars.E2E_TEST_GRAFANA_DASHBOARD_URL || '/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs' }} E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} E2E_TEST_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} E2E_TEST_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }} @@ -689,12 +689,13 @@ jobs: E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary" - E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + E2E_TEST_GRAFANA_DASHBOARD_URL: ${{ matrix.tests.test_env_vars.E2E_TEST_GRAFANA_DASHBOARD_URL || '/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs' }} E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} E2E_TEST_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} E2E_TEST_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }} E2E_TEST_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }} E2E_TEST_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }} + DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable with: test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: make gomod @@ -756,7 +757,7 @@ jobs: - name: Send Slack notification uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 - if: ${{ inputs.slack_notification_after_tests }} + if: ${{ inputs.slack_notification_after_tests == 'true' || inputs.slack_notification_after_tests == 'always' || (inputs.slack_notification_after_tests == 'on_failure' && contains(join(needs.*.result, ','), 'failure')) }} id: slack env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}