automation-nightly-tests.yml

name: Automation Nightly Tests
on:
  schedule:
    - cron: "0 0 * * *" # Run nightly
  push:
    tags:
      - "*"
  workflow_dispatch:

env:
  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink

jobs:
  build-chainlink:
    environment: integration
    permissions:
      id-token: write
      contents: read
    name: Build Chainlink Image
    runs-on: ubuntu20.04-16cores-64GB
    steps:
      - name: Collect Metrics
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@0281b09807758be1dcc41651e44e62b353808c47 # v2.1.0
        with:
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Build Chainlink Image
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Build Chainlink Image
        uses: ./.github/actions/build-chainlink-image
        with:
          tag_suffix: ""
          dockerfile: core/chainlink.Dockerfile
          git_commit_sha: ${{ github.sha }}
          GRAFANA_CLOUD_BASIC_AUTH: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
          GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }}
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

  automation-upgrade-tests:
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs: [build-chainlink]
    env:
      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      TEST_LOG_LEVEL: info
      SELECTED_NETWORKS: "SIMULATED"
    strategy:
      fail-fast: false
      matrix:
        tests:
          - name: Upgrade 2.0
            suite: smoke
            nodes: 1
            os: ubuntu20.04-8cores-32GB
            network: SIMULATED
            command: -run ^TestAutomationNodeUpgrade/registry_2_0 ./smoke
          - name: Upgrade 2.1
            suite: smoke
            nodes: 5
            os: ubuntu20.04-8cores-32GB
            network: SIMULATED
            command: -run ^TestAutomationNodeUpgrade/registry_2_1 ./smoke
          - name: Upgrade 2.2
            suite: smoke
            nodes: 5
            os: ubuntu20.04-8cores-32GB
            network: SIMULATED
            command: -run ^TestAutomationNodeUpgrade/registry_2_2 ./smoke
    runs-on: ${{ matrix.tests.os }}
    name: Automation ${{ matrix.tests.name }} Test
    steps:
      - name: Checkout the repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.head_ref || github.ref_name }}
      - name: Prepare Base64 TOML override
        uses: ./.github/actions/setup-create-base64-upgrade-config
        with:
          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
          chainlinkImage: "public.ecr.aws/chainlink/chainlink"
          chainlinkVersion: "latest"
          upgradeImage: ${{ env.CHAINLINK_IMAGE }}
          upgradeVersion: ${{ github.sha }}
      - name: Run Tests
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@c67a09566412d153ff7640d99f96b43aa03abc04 # v2.3.6
        env:
          TEST_SUITE: ${{ matrix.tests.suite }}
        with:
          test_command_to_run: cd ./integration-tests && go test -timeout 60m -count=1 -json -test.parallel=${{ matrix.tests.nodes }} ${{ matrix.tests.command }} 2>&1 | tee /tmp/gotest.log | gotestfmt
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          cl_repo: 'public.ecr.aws/chainlink/chainlink'
          cl_image_tag: 'latest'
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          artifacts_location: ./integration-tests/${{ matrix.tests.suite }}/logs
          publish_check_name: Automation Results ${{ matrix.tests.name }}
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      - name: Upload test log
        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
        if: failure()
        with:
          name: test-log-${{ matrix.tests.name }}
          path: /tmp/gotest.log
          retention-days: 7
        continue-on-error: true
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@0281b09807758be1dcc41651e44e62b353808c47 # v2.1.0
        with:
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Automation ${{ matrix.tests.name }} Test
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true

  test-notify:
    name: Start Slack Thread
    if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
    environment: integration
    outputs:
      thread_ts: ${{ steps.slack.outputs.thread_ts }}
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    needs: [ automation-upgrade-tests ]
    steps:
      - name: Debug Result
        run: echo ${{ join(needs.*.result, ',') }}
      - name: Main Slack Notification
        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
        id: slack
        with:
          channel-id: C03KJ5S7KEK
          payload: |
            {
              "attachments": [
                {
                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}",
                  "blocks": [
                    {
                      "type": "header",
                      "text": {
                        "type": "plain_text",
                        "text": "Automation Nightly Tests ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}",
                        "emoji": true
                      }
                    },
                    {
                      "type": "divider"
                    },
                    {
                      "type": "section",
                      "text": {
                        "type": "mrkdwn",
                        "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>"
                      }
                    }
                  ]
                }
              ]
            }
        env:
          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

  test-results:
    name: Post Test Results for ${{ matrix.name }}
    if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    needs: test-notify
    strategy:
      fail-fast: false
      matrix:
        name: [ Upgrade 2.0, Upgrade 2.1, Upgrade 2.2
    steps:
      - name: Get Results
        id: test-results
        run: |
          # I feel like there's some clever, fully jq way to do this, but I ain't got the motivation to figure it out
          echo "Querying test results"

          PARSED_RESULTS=$(curl \
          -H "Authorization: Bearer ${{ github.token }}" \
          'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \
          | jq -r --arg pattern "${{ matrix.name }} Test" '.jobs[]
          | select(.name | test($pattern)) as $job
          | $job.steps[]
          | select(.name == "Run Tests")
          | { conclusion: (if .conclusion == "success" then ":white_check_mark:" else ":x:" end), product: ("*" + ($job.name | capture($pattern).product) + "*") }')

          echo "Parsed Results:"
          echo $PARSED_RESULTS

          ALL_SUCCESS=true
          for row in $(echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'); do
              success=false
              break
          done

          echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT

          FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[] 
          | { 
            conclusion: .conclusion, 
            product: .product 
            }
          ]
          | map("{\"type\": \"section\", \"text\": {\"type\": \"mrkdwn\", \"text\": \"\(.product): \(.conclusion)\"}}")
          | join(",")')

          echo "Formatted Results:"
          echo $FORMATTED_RESULTS

          # Cleans out backslashes and quotes from jq
          CLEAN_RESULTS=$(echo "$FORMATTED_RESULTS" | sed 's/\\\"/"/g' | sed 's/^"//;s/"$//')

          echo "Clean Results"
          echo $CLEAN_RESULTS

          echo results=$CLEAN_RESULTS >> $GITHUB_OUTPUT

      - name: Test Details
        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
        with:
          channel-id: C03KJ5S7KEK
          payload: |
            {
              "thread_ts": "${{ needs.test-notify.outputs.thread_ts }}",
              "attachments": [
                {
                  "color": "${{ steps.test-results.outputs.all_success && '#2E7D32' || '#C62828' }}",
                  "blocks": [
                    {
                      "type": "header",
                      "text": {
                        "type": "plain_text",
                        "text": "${{ matrix.name }} ${{ steps.test-results.outputs.all_success && ':white_check_mark:' || ':x: Notifying <@U02Q14G80TY>'}}",
                        "emoji": true
                      }
                    },
                    {
                      "type": "divider"
                    },
                    ${{ steps.test-results.outputs.results }}
                  ]
                }
              ]
            }
        env:
          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}