Migrate more e2e test workflows to use the reusable workflow

.github/e2e-tests.yml

@@ -164,6 +164,17 @@ runner-test-matrix:
       E2E_TEST_CHAINLINK_VERSION: '{{ env.DEFAULT_CHAINLINK_PLUGINS_VERSION }}' # This is the chainlink version that has the plugins
       ENABLE_OTEL_TRACES: true
 
+  - id: chaos/ocr_chaos_test.go
+    path: integration-tests/chaos/ocr_chaos_test.go
+    test_env_type: k8s-remote-runner
+    runs_on: ubuntu-latest
+    workflows:
+      - Automation On Demand Tests
+      - E2E Chaos Tests
+    test_cmd: cd integration-tests/chaos && DETACH_RUNNER=false go test -test.run "^TestOCRChaos$" -v -test.parallel=10 -timeout 60m -count=1 -json
+    test_env_vars:
+      TEST_SUITE: chaos
+
   # END: OCR tests
 
   # START: Automation tests
@@ -531,7 +542,7 @@ runner-test-matrix:
     test_env_vars:
       TEST_SUITE: reorg
     workflows:
-      - Run Automation On Demand Tests (TEST WORKFLOW)
+      - Automation On Demand Tests
     test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_0 -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-automation-on-demand-reorg
 
@@ -542,7 +553,7 @@ runner-test-matrix:
     test_env_vars:
       TEST_SUITE: reorg
     workflows:
-      - Run Automation On Demand Tests (TEST WORKFLOW)
+      - Automation On Demand Tests
     test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_1 -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-automation-on-demand-reorg
 
@@ -553,7 +564,7 @@ runner-test-matrix:
     test_env_vars:
       TEST_SUITE: reorg
     workflows:
-      - Run Automation On Demand Tests (TEST WORKFLOW)
+      - Automation On Demand Tests
     test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_2 -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-automation-on-demand-reorg
 
@@ -564,7 +575,7 @@ runner-test-matrix:
     test_env_vars:
       TEST_SUITE: reorg
     workflows:
-      - Run Automation On Demand Tests (TEST WORKFLOW)
+      - Automation On Demand Tests
     test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_3 -test.parallel=2 -timeout 30m -count=1 -json
     pyroscope_env: ci-automation-on-demand-reorg
 
@@ -573,7 +584,8 @@ runner-test-matrix:
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
     workflows:
-      - Run Automation On Demand Tests (TEST WORKFLOW)
+      - Automation On Demand Tests
+      - E2E Chaos Tests
     test_cmd: cd integration-tests/chaos && DETACH_RUNNER=false go test -v -test.run ^TestAutomationChaos$ -test.parallel=20 -timeout 60m -count=1 -json
     pyroscope_env: ci-automation-on-demand-chaos
     test_env_vars:
@@ -964,4 +976,30 @@ runner-test-matrix:
     test_env_vars:
       E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2      
 
+  - id: ccip-tests/chaos/ccip_test.go
+    path: integration-tests/ccip-tests/chaos/ccip_test.go
+    test_env_type: k8s-remote-runner
+    runs_on: ubuntu-latest
+    workflows:
+      - E2E CCIP Chaos Tests
+    test_cmd: cd integration-tests/ccip-tests/chaos && DETACH_RUNNER=false go test ccip_test.go -v -test.parallel=11 -timeout 60m -count=1 -json
+    test_env_vars:
+      TEST_SUITE: chaos
+      TEST_TRIGGERED_BY: ccip-cron-chaos-eth
+      TEST_LOG_LEVEL: debug
+
+  - id: ccip-tests/load/ccip_test.go:^TestLoadCCIPStableWithPodChaosDiffCommitAndExec
+    path: integration-tests/ccip-tests/load/ccip_test.go
+    test_env_type: k8s-remote-runner
+    runs_on: ubuntu-latest
+    workflows:
+      # Disabled until CCIP-2555 is resolved
+      # - E2E CCIP Chaos Tests
+    test_cmd: cd integration-tests/ccip-tests/load && DETACH_RUNNER=false go test -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' -v -test.parallel=4 -timeout 120m -count=1 -json
+    test_env_vars:
+      TEST_SUITE: chaos
+      TEST_TRIGGERED_BY: ccip-cron-chaos-eth
+      TEST_LOG_LEVEL: debug
+      E2E_TEST_GRAFANA_DASHBOARD_URL: /d/6vjVx-1V8/ccip-long-running-tests
+
   # END: CCIP tests

.github/workflows/ccip-chaos-tests.yml

@@ -11,248 +11,31 @@ concurrency:
   group: chaos-ccip-tests-chainlink-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  # TODO: TT-1470 - Update image names as we solidify new realease strategy
-  CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-  ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
-  MOD_CACHE_VERSION: 1
-
 jobs:
-  build-chainlink:
-    environment: integration
-    permissions:
-      id-token: write
-      contents: read
-    name: Build Chainlink Image
-    runs-on: ubuntu20.04-16cores-64GB
-    steps:
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Check if image exists
-        id: check-image
-        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
-        with:
-          repository: chainlink
-          tag: ${{ github.sha }}
-          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      - name: Build Image
-        if: steps.check-image.outputs.exists == 'false'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          cl_repo: smartcontractkit/chainlink
-          cl_ref: ${{ github.sha }}
-          push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ github.sha }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      - name: Collect Metrics
-        if: always()
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
-        with:
-          id: ccip-chaos-tests-build-chainlink-image
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Build Chainlink Image
-        continue-on-error: true
-
-  build-test-image:
-    environment: integration
-    permissions:
-      id-token: write
-      contents: read
-    name: Build Test Image
-    runs-on: ubuntu20.04-16cores-64GB
-    steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
-        with:
-          id: ccip-chaos-tests-build-test-image
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Build Test Image
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Build Test Image
-        uses: ./.github/actions/build-test-image
-        with:
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-
-  ccip-chaos-tests:
-    environment: integration
-    permissions:
-      issues: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    name: CCIP Chaos Tests
-    runs-on: ubuntu-latest
-    needs: [ build-chainlink, build-test-image ]
-    env:
-      TEST_SUITE: chaos
-      TEST_ARGS: -test.timeout 30m
-      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_TRIGGERED_BY: ccip-cron-chaos-eth
-      TEST_LOG_LEVEL: debug
-      DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
-      GH_TOKEN: ${{ github.token }}
-    steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
-        with:
-          id: ccip-chaos-tests
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: CCIP Chaos Tests
-          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Prepare Base64 TOML override for CCIP secrets
-        uses: ./.github/actions/setup-create-base64-config-ccip
-        id: setup_create_base64_config_ccip
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkVersion: ${{ github.sha }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-      - name: Run Chaos Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26
-        env:
-          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
-          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
-        with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 -run 'TestChaosCCIP' ./chaos 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
-          test_download_vendor_packages_command: make gomod
-          artifacts_location: ./integration-tests/chaos/logs
-          publish_check_name: CCIP Chaos Test Results
-          publish_report_paths: ./tests-chaos-report.xml
-          triggered_by: ${{ env.TEST_TRIGGERED_BY }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-          CGO_ENABLED: "1"
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
-          DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }}
-          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
-          DEFAULT_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-          DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
-          DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-
-      ## Notify in slack if the job fails
-      - name: Notify Slack
-        if: failure() && github.event_name != 'workflow_dispatch'
-        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
-        with:
-          channel-id: "#ccip-testing"
-          slack-message: ":x: :mild-panic-intensifies: CCIP chaos tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}"
-      ## Run Cleanup if the job succeeds
-      - name: cleanup
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
-        with:
-          triggered_by: ${{ env.TEST_TRIGGERED_BY }}
-
-  ccip-chaos-with-load-tests:
-    environment: integration
-    permissions:
-      issues: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    name: CCIP Load With Chaos Tests
-    if: false # Disabled until CCIP-2555 is resolved
-    runs-on: ubuntu-latest
-    needs: [ build-chainlink, build-test-image ]
-    env:
-      TEST_SUITE: load
-      TEST_ARGS: -test.timeout 1h
-      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_TRIGGERED_BY: ccip-cron-chaos-and-load-eth
-      TEST_LOG_LEVEL: debug
-      DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
-      GH_TOKEN: ${{ github.token }}
-    steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
-        with:
-          id: ccip-chaos-tests-with-load-test
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: CCIP load with chaos test
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Prepare Base64 TOML override for CCIP secrests
-        uses: ./.github/actions/setup-create-base64-config-ccip
-        id: setup_create_base64_config_ccip
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkVersion: ${{ github.sha }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-      - name: Run Load With Chaos Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26
-        env:
-          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
-          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
-        with:
-          test_command_to_run: cd ./integration-tests/ccip-tests && go test -timeout 2h -count=1 -json -test.parallel 4 -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' ./load 2>&1 | tee /tmp/gotest.log | gotestfmt
-          test_download_vendor_packages_command: make gomod
-          artifacts_location: ./integration-tests/load/logs
-          publish_check_name: CCIP Chaos With Load Test Results
-          publish_report_paths: ./tests-chaos-with-load-report.xml
-          triggered_by: ${{ env.TEST_TRIGGERED_BY }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-          CGO_ENABLED: "1"
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
-          DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }}
-          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
-          DEFAULT_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-          DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
-          DEFAULT_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
-      ## Notify in slack if the job fails
-      - name: Notify Slack
-        if: failure() && github.event_name != 'workflow_dispatch'
-        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
-        with:
-          channel-id: "#ccip-testing"
-          slack-message: ":x: :mild-panic-intensifies: CCIP chaos with load tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}"
-      ## Run Cleanup if the job succeeds
-      - name: cleanup
-        if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
-        with:
-          triggered_by: ${{ env.TEST_TRIGGERED_BY }}
+  run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      chainlink_version: ${{ github.sha }}
+      require_chainlink_image_versions_in_qa_ecr: ${{ github.sha }}
+      test_workflow: E2E CCIP Chaos Tests
+      test_log_level: debug
+      slack_notification_after_tests: on_failure
+      slack_notification_after_tests_channel_id: '#ccip-testing'
+      slack_notification_after_tests_name: CCIP Chaos E2E Tests
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}    
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}