-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Moves Client Compatability Tests to Nightly Run
- Loading branch information
Showing
3 changed files
with
261 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,256 @@ | ||
| name: Client Compatability Tests | ||
| on: | ||
| schedule: | ||
| - cron: "0 5 * * *" # Run every night at midnight EST | ||
| push: | ||
| tags: | ||
| - "*" | ||
| workflow_dispatch: | ||
|
|
||
| jobs: | ||
| build-chainlink: | ||
| environment: integration | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| name: Build Chainlink Image | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Collect Metrics | ||
| id: collect-gha-metrics | ||
| uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 | ||
| with: | ||
| basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} | ||
| hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} | ||
| this-job-name: Build Chainlink Image | ||
| continue-on-error: true | ||
| - name: Checkout the repo | ||
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} | ||
| - name: Build Chainlink Image | ||
| uses: ./.github/actions/build-chainlink-image | ||
| with: | ||
| tag_suffix: "" | ||
| dockerfile: core/chainlink.Dockerfile | ||
| git_commit_sha: ${{ github.sha }} | ||
| GRAFANA_CLOUD_BASIC_AUTH: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} | ||
| GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }} | ||
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | ||
| AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | ||
|
|
||
| client-compatability-matrix: | ||
| environment: integration | ||
| permissions: | ||
| checks: write | ||
| pull-requests: write | ||
| id-token: write | ||
| contents: read | ||
| needs: [build-chainlink] | ||
| env: | ||
| SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 | ||
| CHAINLINK_COMMIT_SHA: ${{ github.sha }} | ||
| CHAINLINK_ENV_USER: ${{ github.actor }} | ||
| TEST_LOG_LEVEL: debug | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| product: | ||
| - name: ocr-geth | ||
| os: ubuntu-latest | ||
| run: -run TestOCRBasic | ||
| file: ocr | ||
| client: geth | ||
| pyroscope_env: ci-smoke-ocr-evm-simulated | ||
| # Uncomment, when https://smartcontract-it.atlassian.net/browse/TT-753 is DONE | ||
| # - name: ocr-nethermind | ||
| # run: -run TestOCRBasic | ||
| # file: ocr | ||
| # client: nethermind | ||
| # pyroscope_env: ci-smoke-ocr-evm-simulated | ||
| - name: ocr-besu | ||
| run: -run TestOCRBasic | ||
| file: ocr | ||
| client: besu | ||
| pyroscope_env: ci-smoke-ocr-evm-simulated | ||
| - name: ocr-erigon | ||
| run: -run TestOCRBasic | ||
| file: ocr | ||
| client: erigon | ||
| pyroscope_env: ci-smoke-ocr-evm-simulated | ||
| - name: ocr2-geth | ||
| run: -run TestOCRv2Basic | ||
| file: ocr2 | ||
| client: geth | ||
| pyroscope_env: ci-smoke-ocr2-evm-simulated | ||
| # Uncomment, when https://smartcontract-it.atlassian.net/browse/TT-753 is DONE | ||
| # - name: ocr2-nethermind | ||
| # run: -run TestOCRv2Basic | ||
| # file: ocr2 | ||
| # client: nethermind | ||
| # pyroscope_env: ci-smoke-ocr2-evm-simulated | ||
| - name: ocr2-besu | ||
| run: -run TestOCRv2Basic | ||
| file: ocr2 | ||
| client: besu | ||
| pyroscope_env: ci-smoke-ocr2-evm-simulated | ||
| - name: ocr2-erigon | ||
| run: -run TestOCRv2Basic | ||
| file: ocr2 | ||
| client: erigon | ||
| pyroscope_env: ci-smoke-ocr2-evm-simulated | ||
| runs-on: ubuntu-latest | ||
| name: ETH Smoke Tests ${{ matrix.product.name }} | ||
| steps: | ||
| - name: Checkout the repo | ||
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} | ||
| - name: Build Go Test Command | ||
| id: build-go-test-command | ||
| run: | | ||
| # if the matrix.product.run is set, use it for a different command | ||
| if [ "${{ matrix.product.run }}" != "" ]; then | ||
| echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" | ||
| else | ||
| echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" | ||
| fi | ||
| - name: Check for "enable tracing" label | ||
| id: check-label | ||
| run: | | ||
| label=$(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH") | ||
| if [[ -n "$label" ]]; then | ||
| if [[ "$label" == "enable tracing" ]]; then | ||
| echo "Enable tracing label found." | ||
| echo "trace=true" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "Enable tracing label not found." | ||
| echo "trace=false" >> $GITHUB_OUTPUT | ||
| fi | ||
| else | ||
| echo "No labels present or labels are null." | ||
| echo "trace=false" >> $GITHUB_OUTPUT | ||
| fi | ||
| - name: Setup Grafana and OpenTelemetry | ||
| id: docker-setup | ||
| if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' | ||
| run: | | ||
| # Create network | ||
| docker network create --driver bridge tracing | ||
| # Make trace directory | ||
| cd integration-tests/smoke/ | ||
| mkdir ./traces | ||
| chmod -R 777 ./traces | ||
| # Switch directory | ||
| cd ../../.github/tracing | ||
| # Create a Docker volume for traces | ||
| # docker volume create otel-traces | ||
| # Start OpenTelemetry Collector | ||
| # Note the user must be set to the same user as the runner for the trace data to be accessible | ||
| docker run -d --network=tracing --name=otel-collector \ | ||
| -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \ | ||
| -v $PWD/../../integration-tests/smoke/traces:/tracing \ | ||
| --user "$(id -u):$(id -g)" \ | ||
| -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml | ||
| - name: Locate Docker Volume | ||
| id: locate-volume | ||
| if: false | ||
| run: | | ||
| echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT | ||
| - name: Show Otel-Collector Logs | ||
| if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' | ||
| run: | | ||
| docker logs otel-collector | ||
| ## Run this step when changes that require tests to be run are made | ||
| - name: Run Tests | ||
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 | ||
| env: | ||
| PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 | ||
| PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} | ||
| PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} | ||
| ETH2_EL_CLIENT: ${{matrix.product.client}} | ||
| LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} | ||
| LOKI_URL: ${{ secrets.LOKI_URL }} | ||
| LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} | ||
| LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} | ||
| GRAFANA_URL: ${{ vars.GRAFANA_URL }} | ||
| GRAFANA_DATASOURCE: ${{ vars.GRAFANA_DATASOURCE }} | ||
| RUN_ID: ${{ github.run_id }} | ||
| with: | ||
| test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt | ||
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | ||
| cl_repo: ${{ env.CHAINLINK_IMAGE }} | ||
| cl_image_tag: ${{ github.sha }}${{ matrix.product.tag_suffix }} | ||
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | ||
| artifacts_name: ${{ matrix.product.name }}-test-logs | ||
| artifacts_location: ./integration-tests/smoke/logs/ | ||
| publish_check_name: ${{ matrix.product.name }} | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| go_mod_path: ./integration-tests/go.mod | ||
| cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} | ||
| cache_restore_only: "true" | ||
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | ||
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | ||
| QA_KUBECONFIG: "" | ||
| ## Run this step when changes that do not need the test to run are made | ||
| - name: Run Setup | ||
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 | ||
| with: | ||
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | ||
| go_mod_path: ./integration-tests/go.mod | ||
| cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} | ||
| cache_restore_only: "true" | ||
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | ||
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | ||
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | ||
| - name: Show Otel-Collector Logs | ||
| if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' | ||
| run: | | ||
| docker logs otel-collector | ||
| - name: Collect Metrics | ||
| if: always() | ||
| id: collect-gha-metrics | ||
| uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 | ||
| with: | ||
| basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} | ||
| hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} | ||
| this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} | ||
| test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' | ||
| continue-on-error: true | ||
| - name: Permissions on traces | ||
| if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' | ||
| run: | | ||
| ls -l ./integration-tests/smoke/traces | ||
| - name: Upload Trace Data | ||
| if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: trace-data | ||
| path: ./integration-tests/smoke/traces/trace-data.json | ||
| - name: Print failed test summary | ||
| if: always() | ||
| run: | | ||
| directory="./integration-tests/smoke/.test_summary" | ||
| files=("$directory"/*) | ||
| if [ -d "$directory" ]; then | ||
| echo "Test summary folder found" | ||
| if [ ${#files[@]} -gt 0 ]; then | ||
| first_file="${files[0]}" | ||
| echo "Name of the first test summary file: $(basename "$first_file")" | ||
| echo "### Failed Test Execution Logs Dashboard (over VPN):" >> $GITHUB_STEP_SUMMARY | ||
| cat "$first_file" | jq -r '.loki[] | "* [\(.test_name)](\(.value))"' >> $GITHUB_STEP_SUMMARY | ||
| if [ ${#files[@]} -gt 1 ]; then | ||
| echo "Found more than one test summary file. This is incorrect, there should be only one file" | ||
| fi | ||
| else | ||
| echo "Test summary directory is empty. This should not happen" | ||
| fi | ||
| else | ||
| echo "No test summary folder found. If no test failed or log collection wasn't explicitly requested this is correct. Exiting" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters