From c3ba3a503b8a3e7a1fab0567e0a1cd4b630505d5 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 21 Dec 2023 15:37:34 -0500 Subject: [PATCH] Fixes Compatibility Tests (#11652) * Cleanup * Fix title * Fix CL image * Cleanup and convert to build binary * Words are hard --- .../workflows/client-compatibility-tests.yml | 157 +++++++++--------- .github/workflows/integration-tests.yml | 10 +- .github/workflows/live-testnet-tests.yml | 2 +- 3 files changed, 76 insertions(+), 93 deletions(-) diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml index 8688e51e1df..865c72258cd 100644 --- a/.github/workflows/client-compatibility-tests.yml +++ b/.github/workflows/client-compatibility-tests.yml @@ -7,7 +7,14 @@ on: - "*" workflow_dispatch: +env: + CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com + MOD_CACHE_VERSION: 2 + jobs: + # Build Test Dependencies + build-chainlink: environment: integration permissions: @@ -38,6 +45,38 @@ jobs: GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }} AWS_REGION: ${{ secrets.QA_AWS_REGION }} AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + + build-tests: + environment: integration + permissions: + id-token: write + contents: read + name: Build Tests Binary + runs-on: ubuntu-latest + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: Build Tests Binary + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Build Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-tests@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 + with: + test_download_vendor_packages_command: cd ./integration-tests && go mod download + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + binary_name: tests + + # End Build Test Dependencies client-compatibility-matrix: environment: integration @@ -46,7 +85,7 @@ jobs: pull-requests: write id-token: write contents: read - needs: [build-chainlink] + needs: [build-chainlink, build-tests] env: SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 CHAINLINK_COMMIT_SHA: ${{ github.sha }} @@ -55,127 +94,79 @@ jobs: strategy: fail-fast: false matrix: - product: + include: - name: ocr-geth os: ubuntu-latest - run: -run TestOCRBasic + test: TestOCRBasic file: ocr client: geth - pyroscope_env: ci-smoke-ocr-evm-simulated + pyroscope_env: ci-smoke-ocr-geth-simulated # Uncomment, when https://smartcontract-it.atlassian.net/browse/TT-753 is DONE # - name: ocr-nethermind - # run: -run TestOCRBasic + # test: TestOCRBasic # file: ocr # client: nethermind - # pyroscope_env: ci-smoke-ocr-evm-simulated + # pyroscope_env: ci-smoke-ocr-nethermind-simulated - name: ocr-besu - run: -run TestOCRBasic + test: TestOCRBasic file: ocr client: besu - pyroscope_env: ci-smoke-ocr-evm-simulated + pyroscope_env: ci-smoke-ocr-besu-simulated - name: ocr-erigon - run: -run TestOCRBasic + test: TestOCRBasic file: ocr client: erigon - pyroscope_env: ci-smoke-ocr-evm-simulated + pyroscope_env: ci-smoke-ocr-erigon-simulated - name: ocr2-geth - run: -run TestOCRv2Basic + test: TestOCRv2Basic file: ocr2 client: geth - pyroscope_env: ci-smoke-ocr2-evm-simulated + pyroscope_env: ci-smoke-ocr2-geth-simulated # Uncomment, when https://smartcontract-it.atlassian.net/browse/TT-753 is DONE # - name: ocr2-nethermind - # run: -run TestOCRv2Basic + # test: TestOCRv2Basic # file: ocr2 # client: nethermind - # pyroscope_env: ci-smoke-ocr2-evm-simulated + # pyroscope_env: ci-smoke-nethermind-evm-simulated - name: ocr2-besu - run: -run TestOCRv2Basic + test: TestOCRv2Basic file: ocr2 client: besu - pyroscope_env: ci-smoke-ocr2-evm-simulated + pyroscope_env: ci-smoke-ocr2-besu-simulated - name: ocr2-erigon - run: -run TestOCRv2Basic + test: TestOCRv2Basic file: ocr2 client: erigon - pyroscope_env: ci-smoke-ocr2-evm-simulated + pyroscope_env: ci-smoke-ocr2-erigon-simulated runs-on: ubuntu-latest - name: Client Compatibility Test ${{ matrix.product.name }} + name: Client Compatibility Test ${{ matrix.name }} steps: - - name: Checkout the repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Download Tests Binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: - ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Build Go Test Command - id: build-go-test-command - run: | - # if the matrix.product.run is set, use it for a different command - if [ "${{ matrix.product.run }}" != "" ]; then - echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" - else - echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" - fi + name: tests - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@912bed7e07a1df4d06ea53a031e9773bb65dc7bd # v2.3.0 env: - PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} + PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + PYROSCOPE_ENVIRONMENT: ci-client-compatability-${{ matrix.client }}-testnet PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - ETH2_EL_CLIENT: ${{matrix.product.client}} - LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} - LOKI_URL: ${{ secrets.LOKI_URL }} - LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} - LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} - GRAFANA_URL: ${{ vars.GRAFANA_URL }} - GRAFANA_DATASOURCE: ${{ vars.GRAFANA_DATASOURCE }} - RUN_ID: ${{ github.run_id }} + ETH2_EL_CLIENT: ${{matrix.client}} with: - test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt - test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_command_to_run: ./tests -test.timeout 30m -test.run ${{ matrix.test }} + binary_name: tests cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ github.sha }}${{ matrix.product.tag_suffix }} + cl_image_tag: ${{ github.sha }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - artifacts_name: ${{ matrix.product.name }}-test-logs - artifacts_location: ./integration-tests/smoke/logs/ - publish_check_name: ${{ matrix.product.name }} + dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }} + dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }} + artifacts_location: ./logs token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: "" - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 - with: - basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} - this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} - test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - continue-on-error: true - - name: Print failed test summary - if: always() - run: | - directory="./integration-tests/smoke/.test_summary" - files=("$directory"/*) - if [ -d "$directory" ]; then - echo "Test summary folder found" - if [ ${#files[@]} -gt 0 ]; then - first_file="${files[0]}" - echo "Name of the first test summary file: $(basename "$first_file")" - echo "### Failed Test Execution Logs Dashboard (over VPN):" >> $GITHUB_STEP_SUMMARY - cat "$first_file" | jq -r '.loki[] | "* [\(.test_name)](\(.value))"' >> $GITHUB_STEP_SUMMARY - if [ ${#files[@]} -gt 1 ]; then - echo "Found more than one test summary file. This is incorrect, there should be only one file" - fi - else - echo "Test summary directory is empty. This should not happen" - fi - else - echo "No test summary folder found. If no test failed or log collection wasn't explicitly requested this is correct. Exiting" - fi + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} start-slack-thread: name: Start Slack Thread @@ -208,7 +199,7 @@ jobs: "type": "header", "text": { "type": "plain_text", - "text": "Live Smoke Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", + "text": "Client Compatability Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", "emoji": true } }, @@ -262,7 +253,7 @@ jobs: github_token: ${{ github.token }} github_repository: ${{ github.repository }} workflow_run_id: ${{ github.run_id }} - github_job_name_regex: ^Client Compatability Test ${{ matrix.product }}-(?.*?)$ + github_job_name_regex: ^Client Compatibility Test ${{ matrix.product }}-(?.*?)$ message_title: ${{ matrix.product }} slack_channel_id: ${{ secrets.QA_SLACK_CHANNEL }} slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 08e24fc40a9..e588ab509f2 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -325,21 +325,13 @@ jobs: - name: ocr nodes: 1 os: ubuntu-latest - run: -run TestOCRJobReplacement file: ocr pyroscope_env: ci-smoke-ocr-evm-simulated - name: ocr2 nodes: 1 os: ubuntu-latest - run: -run TestOCRv2JobReplacement file: ocr2 - pyroscope_env: ci-smoke-ocr2-evm-simulated - - name: ocr2 - nodes: 1 - os: ubuntu-latest - run: -run TestOCRv2Basic - file: ocr2 - pyroscope_env: ci-smoke-ocr2-evm-simulated + pyroscope_env: ci-smoke-ocr2-evm-simulated - name: ocr2 nodes: 1 os: ubuntu-latest diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml index 2b5ecb2d5da..3298c45813a 100644 --- a/.github/workflows/live-testnet-tests.yml +++ b/.github/workflows/live-testnet-tests.yml @@ -66,7 +66,7 @@ env: jobs: # Build Test Dependencies - + build-chainlink: environment: integration permissions: