Fixes Compatibility Tests (#11652)

* Cleanup * Fix title * Fix CL image * Cleanup and convert to build binary * Words are hard
smartcontractkit · Dec 21, 2023 · c3ba3a5 · c3ba3a5
1 parent 7236361
commit c3ba3a5
Show file tree

Hide file tree

Showing 3 changed files with 76 additions and 93 deletions.
diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml
@@ -7,7 +7,14 @@ on:
       - "*"
   workflow_dispatch:
 
+env:
+  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+  INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
+  MOD_CACHE_VERSION: 2
+
 jobs:
+  # Build Test Dependencies
+
   build-chainlink:
     environment: integration
     permissions:
@@ -38,6 +45,38 @@ jobs:
           GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }}
           AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+
+  build-tests:
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    name: Build Tests Binary
+    runs-on: ubuntu-latest
+    steps:
+      - name: Collect Metrics
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
+        with:
+          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          this-job-name: Build Tests Binary
+        continue-on-error: true
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
+      - name: Build Tests
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-tests@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0
+        with:
+          test_download_vendor_packages_command: cd ./integration-tests && go mod download
+          token: ${{ secrets.GITHUB_TOKEN }}
+          go_mod_path: ./integration-tests/go.mod
+          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
+          cache_restore_only: "true"
+          binary_name: tests
+
+  # End Build Test Dependencies
 
   client-compatibility-matrix:
     environment: integration
@@ -46,7 +85,7 @@ jobs:
       pull-requests: write
       id-token: write
       contents: read
-    needs: [build-chainlink]
+    needs: [build-chainlink, build-tests]
     env:
       SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
       CHAINLINK_COMMIT_SHA: ${{ github.sha }}
@@ -55,127 +94,79 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        product:
+        include:
           - name: ocr-geth
             os: ubuntu-latest
-            run: -run TestOCRBasic
+            test: TestOCRBasic
             file: ocr
             client: geth
-            pyroscope_env: ci-smoke-ocr-evm-simulated         
+            pyroscope_env: ci-smoke-ocr-geth-simulated         
           # Uncomment, when https://smartcontract-it.atlassian.net/browse/TT-753 is DONE
           # - name: ocr-nethermind
-          #   run: -run TestOCRBasic
+          #   test: TestOCRBasic
           #   file: ocr
           #   client: nethermind
-          #   pyroscope_env: ci-smoke-ocr-evm-simulated                          
+          #   pyroscope_env: ci-smoke-ocr-nethermind-simulated                          
           - name: ocr-besu
-            run: -run TestOCRBasic
+            test: TestOCRBasic
             file: ocr
             client: besu
-            pyroscope_env: ci-smoke-ocr-evm-simulated        
+            pyroscope_env: ci-smoke-ocr-besu-simulated        
           - name: ocr-erigon
-            run: -run TestOCRBasic
+            test: TestOCRBasic
             file: ocr
             client: erigon
-            pyroscope_env: ci-smoke-ocr-evm-simulated         
+            pyroscope_env: ci-smoke-ocr-erigon-simulated         
           - name: ocr2-geth
-            run: -run TestOCRv2Basic
+            test: TestOCRv2Basic
             file: ocr2
             client: geth
-            pyroscope_env: ci-smoke-ocr2-evm-simulated            
+            pyroscope_env: ci-smoke-ocr2-geth-simulated            
           # Uncomment, when https://smartcontract-it.atlassian.net/browse/TT-753 is DONE            
           # - name: ocr2-nethermind
-          #   run: -run TestOCRv2Basic
+          #   test: TestOCRv2Basic
           #   file: ocr2
           #   client: nethermind
-          #   pyroscope_env: ci-smoke-ocr2-evm-simulated                         
+          #   pyroscope_env: ci-smoke-nethermind-evm-simulated                         
           - name: ocr2-besu
-            run: -run TestOCRv2Basic
+            test: TestOCRv2Basic
             file: ocr2
             client: besu            
-            pyroscope_env: ci-smoke-ocr2-evm-simulated      
+            pyroscope_env: ci-smoke-ocr2-besu-simulated      
           - name: ocr2-erigon
-            run: -run TestOCRv2Basic
+            test: TestOCRv2Basic
             file: ocr2
             client: erigon            
-            pyroscope_env: ci-smoke-ocr2-evm-simulated                           
+            pyroscope_env: ci-smoke-ocr2-erigon-simulated                           
     runs-on: ubuntu-latest
-    name: Client Compatibility Test ${{ matrix.product.name }}
+    name: Client Compatibility Test ${{ matrix.name }}
     steps:
-      - name: Checkout the repo
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Download Tests Binary
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
-          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
-      - name: Build Go Test Command
-        id: build-go-test-command
-        run: |
-          # if the matrix.product.run is set, use it for a different command
-          if [ "${{ matrix.product.run }}" != "" ]; then
-            echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT"
-          else
-            echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
-          fi
+          name: tests
       - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0
         env:
-          PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }}
+          PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+          PYROSCOPE_ENVIRONMENT: ci-client-compatability-${{ matrix.client }}-testnet
           PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-          ETH2_EL_CLIENT: ${{matrix.product.client}}
-          LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
-          LOKI_URL: ${{ secrets.LOKI_URL }}
-          LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
-          LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          GRAFANA_URL: ${{ vars.GRAFANA_URL }}
-          GRAFANA_DATASOURCE: ${{ vars.GRAFANA_DATASOURCE }}
-          RUN_ID: ${{ github.run_id }}
+          ETH2_EL_CLIENT: ${{matrix.client}}
         with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt
-          test_download_vendor_packages_command: cd ./integration-tests && go mod download
+          test_command_to_run: ./tests -test.timeout 30m -test.run ${{ matrix.test }}
+          binary_name: tests
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ github.sha }}${{ matrix.product.tag_suffix }}
+          cl_image_tag: ${{ github.sha }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          artifacts_name: ${{ matrix.product.name }}-test-logs
-          artifacts_location: ./integration-tests/smoke/logs/
-          publish_check_name: ${{ matrix.product.name }}
+          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
+          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
+          artifacts_location: ./logs
           token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ""
-      - name: Collect Metrics
-        if: always()
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
-        with:
-          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
-          this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
-          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
-        continue-on-error: true
-      - name: Print failed test summary
-        if: always()
-        run: |
-          directory="./integration-tests/smoke/.test_summary"
-          files=("$directory"/*)
-          if [ -d "$directory" ]; then
-            echo "Test summary folder found"
-            if [ ${#files[@]} -gt 0 ]; then
-              first_file="${files[0]}"
-              echo "Name of the first test summary file: $(basename "$first_file")"
-              echo "### Failed Test Execution Logs Dashboard (over VPN):" >> $GITHUB_STEP_SUMMARY
-              cat "$first_file" | jq -r '.loki[] | "* [\(.test_name)](\(.value))"' >> $GITHUB_STEP_SUMMARY
-              if [ ${#files[@]} -gt 1 ]; then
-                echo "Found more than one test summary file. This is incorrect, there should be only one file"
-              fi
-            else
-                echo "Test summary directory is empty. This should not happen"
-            fi
-          else
-            echo "No test summary folder found. If no test failed or log collection wasn't explicitly requested this is correct. Exiting"
-          fi
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
 
   start-slack-thread:
     name: Start Slack Thread
@@ -208,7 +199,7 @@ jobs:
                       "type": "header",
                       "text": {
                         "type": "plain_text",
-                        "text": "Live Smoke Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}",
+                        "text": "Client Compatability Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}",
                         "emoji": true
                       }
                     },
@@ -262,7 +253,7 @@ jobs:
           github_token: ${{ github.token }}
           github_repository: ${{ github.repository }}
           workflow_run_id: ${{ github.run_id }}
-          github_job_name_regex: ^Client Compatability Test ${{ matrix.product }}-(?<cap>.*?)$
+          github_job_name_regex: ^Client Compatibility Test ${{ matrix.product }}-(?<cap>.*?)$
           message_title: ${{ matrix.product }}
           slack_channel_id: ${{ secrets.QA_SLACK_CHANNEL }}
           slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }}

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -325,21 +325,13 @@ jobs:
           - name: ocr
             nodes: 1
             os: ubuntu-latest
-            run: -run TestOCRJobReplacement
             file: ocr
             pyroscope_env: ci-smoke-ocr-evm-simulated                         
           - name: ocr2
             nodes: 1
             os: ubuntu-latest
-            run: -run TestOCRv2JobReplacement
             file: ocr2
-            pyroscope_env: ci-smoke-ocr2-evm-simulated
-          - name: ocr2
-            nodes: 1
-            os: ubuntu-latest
-            run: -run TestOCRv2Basic
-            file: ocr2
-            pyroscope_env: ci-smoke-ocr2-evm-simulated                                                           
+            pyroscope_env: ci-smoke-ocr2-evm-simulated                                                     
           - name: ocr2
             nodes: 1
             os: ubuntu-latest

diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml
@@ -66,7 +66,7 @@ env:
 jobs:
 
   # Build Test Dependencies
-
+  
   build-chainlink:
     environment: integration
     permissions: