From 372ebf06416c6059caeb50dc36b320f7561b6604 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Wed, 20 Nov 2024 14:43:38 -0800 Subject: [PATCH 1/9] feat: conditionally run unit tests --- .../actions/setup-ci-core-tests/action.yml | 109 ++++++++++++ .github/workflows/ci-core-partial.yml | 159 ++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 .github/actions/setup-ci-core-tests/action.yml create mode 100644 .github/workflows/ci-core-partial.yml diff --git a/.github/actions/setup-ci-core-tests/action.yml b/.github/actions/setup-ci-core-tests/action.yml new file mode 100644 index 00000000000..c066c214042 --- /dev/null +++ b/.github/actions/setup-ci-core-tests/action.yml @@ -0,0 +1,109 @@ +name: Setup CI Core Tests +description: Shared setup steps for ci-core +inputs: + test-suite: + description: | + The test suite's name + required: true + + evm-ref-override: + description: | + Overrides the evm/relayer dependency version + required: false + + db-url: + description: | + The expected database URL + required: true + + build-only: + description: | + Only setup the necessary dependencies for building the binary + default: "false" + +runs: + using: composite + steps: + - name: Log Start + shell: bash + run: | + echo "====================================" + echo "Setting up CI Core Tests Environment" + echo "====================================" + + - name: Setup NodeJS + uses: ./.github/actions/setup-nodejs + with: + prod: "true" + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + # only restore for now + restore-build-cache-only: "true" + build-cache-version: ${{ inputs.test-suite }} + + - name: Replace chainlink-evm deps + if: ${{ inputs.evm-ref-override != ''}} + shell: bash + run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref }} + + - name: Setup Solana + uses: ./.github/actions/setup-solana + + - name: Setup wasmd + uses: ./.github/actions/setup-wasmd + + - name: Setup Postgres + if: ${{ inputs.build-only == 'false' }} + uses: ./.github/actions/setup-postgres + + - name: Touching core/web/assets/index.html + if: ${{ inputs.build-only == 'false' }} + shell: bash + run: mkdir -p core/web/assets && touch core/web/assets/index.html + + - name: Download Go vendor packages + shell: bash + run: go mod download + + - name: Go Mod Download (deployment) + if: ${{ matrix.type.test-suite == 'ccip-deployment' }} + shell: bash + working-directory: "./deployment" + run: go mod download + + - name: Build binary + if: ${{ inputs.build-only == 'false' }} + shell: bash + run: go build -o chainlink.test . + + - name: Setup DB + if: ${{ inputs.build-only == 'false' }} + shell: bash + run: ./chainlink.test local db preparetest + env: + CL_DATABASE_URL: ${{ inputs.db-url }} + + - name: Install LOOP Plugins + shell: bash + run: | + pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-feeds) + go install ./cmd/chainlink-feeds + popd + pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-data-streams) + go install ./mercury/cmd/chainlink-mercury + popd + pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-solana) + go install ./pkg/solana/cmd/chainlink-solana + popd + pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-starknet/relayer) + go install ./pkg/chainlink/cmd/chainlink-starknet + popd + + - name: Log End + shell: bash + run: | + echo "=============================================" + echo "Finished Setting up CI Core Tests Environment" + echo "=============================================" diff --git a/.github/workflows/ci-core-partial.yml b/.github/workflows/ci-core-partial.yml new file mode 100644 index 00000000000..8364cc6c25d --- /dev/null +++ b/.github/workflows/ci-core-partial.yml @@ -0,0 +1,159 @@ +name: Core Unit Tests + +# Run on key branches to make sure integration is good, otherwise run on all PR's +on: + push: + branches: + - develop + - main + - "release/*" + merge_group: + pull_request: + +jobs: + run-unit-tests: + name: Tests (${{ matrix.type.test-suite }}) + runs-on: ubuntu22.04-32cores-128GB + permissions: + id-token: write + contents: write + strategy: + fail-fast: false + matrix: + type: + - test-suite: "core" + tag-filter: "" + module-directory: "./" + - test-suite: "integration" + tag-filter: "integration" + module-directory: "./" + - test-suite: "ccip-deployment" + tag-filter: "" + module-directory: "./deployment" + env: + # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests + # when they should not be using it, while still allowing us to DRY up the setup + DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable + steps: + - name: Checkout the repo + uses: actions/checkout@v4.2.1 + + - name: Change Modtime of Files (cache optimization) + shell: bash + run: | + find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true + + - name: Setup CI Core Environment + uses: ./.github/actions/setup-ci-core-tests + with: + test-suite: ${{ matrix.type.cmd }} + db-url: ${{ env.DB_URL }} + + - name: Run Tests + uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison + env: + CL_DATABASE_URL: ${{ env.DB_URL }} + with: + test-suite: ${{ matrix.type.test-suite }} + module-directory: ${{ matrix.type.module-directory }} + tag-filter: ${{ matrix.type.tag-filter }} + update-index: "true" + force-update-index: "true" + build-concurrency: "32" + run-concurrency: "32" + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Print postgres logs + if: ${{ always() }} + run: docker compose logs postgres | tee ../../../postgres_logs.txt + working-directory: ./.github/actions/setup-postgres + + + run-fuzz-tests: + name: Tests (fuzz) + if: ${{ github.ref == 'refs/heads/dont-run' }} + runs-on: ubuntu22.04-32cores-128GB + env: + DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable + steps: + - name: Checkout the repo + uses: actions/checkout@v4.2.1 + + - name: Change Modtime of Files (cache optimization) + shell: bash + run: | + find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true + + - name: Setup CI Core Environment + uses: ./.github/actions/setup-ci-core-tests + with: + test-suite: "fuzz" + db-url: ${{ env.DB_URL }} + + - name: Increase Timeouts + if: ${{ github.event.schedule != ''}} + run: | + echo "FUZZ_TIMEOUT_MINUTES=10">> $GITHUB_ENV + + - name: Run Fuzz Tests + env: + OUTPUT_FILE: ./output.txt + CL_DATABASE_URL: ${{ env.DB_URL }} + run: ./tools/bin/go_core_fuzz ./... + + - name: Print postgres logs + if: ${{ always() }} + run: docker compose logs postgres | tee ../../../postgres_logs.txt + working-directory: ./.github/actions/setup-postgres + + run-race-tests: + name: Tests (race) + if: ${{ github.ref == 'refs/heads/dont-run' }} + runs-on: ubuntu22.04-32cores-128GB + env: + DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable + steps: + - name: Checkout the repo + uses: actions/checkout@v4.2.1 + + - name: Change Modtime of Files (cache optimization) + shell: bash + run: | + find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true + + - name: Setup CI Core Environment + uses: ./.github/actions/setup-ci-core-tests + with: + test-suite: "race" + db-url: ${{ env.DB_URL }} + + - name: Increase Timeouts + if: ${{ github.event.schedule != ''}} + run: | + echo "TIMEOUT=10m" >> $GITHUB_ENV + echo "COUNT=50" >> $GITHUB_ENV + + - name: Run Race Tests + env: + OUTPUT_FILE: ./output.txt + CL_DATABASE_URL: ${{ env.DB_URL }} + run: ./tools/bin/go_core_race_tests ./... + + - name: Print Races + id: print-races + if: ${{ failure() }} + run: | + find race.* | xargs cat > race.txt + if [[ -s race.txt ]]; then + cat race.txt + echo "post_to_slack=true" >> $GITHUB_OUTPUT + else + echo "post_to_slack=false" >> $GITHUB_OUTPUT + fi + echo "github.event_name: ${{ github.event_name }}" + echo "github.ref: ${{ github.ref }}" + + - name: Print postgres logs + if: ${{ always() }} + run: docker compose logs postgres | tee ../../../postgres_logs.txt + working-directory: ./.github/actions/setup-postgres From 9664f4a9e4d67704daf0ec75773b52377410eca8 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Mon, 25 Nov 2024 12:15:36 -0800 Subject: [PATCH 2/9] feat: separate action to different steps --- .github/workflows/ci-core-partial.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-core-partial.yml b/.github/workflows/ci-core-partial.yml index 8364cc6c25d..6da97d5c50a 100644 --- a/.github/workflows/ci-core-partial.yml +++ b/.github/workflows/ci-core-partial.yml @@ -49,19 +49,39 @@ jobs: test-suite: ${{ matrix.type.cmd }} db-url: ${{ env.DB_URL }} + - name: Build Tests + uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison + with: + pipeline-step: "build" + test-suite: ${{ matrix.type.test-suite }} + module-directory: ${{ matrix.type.module-directory }} + tag-filter: ${{ matrix.type.tag-filter }} + build-concurrency: "32" + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Run Tests uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison env: CL_DATABASE_URL: ${{ env.DB_URL }} with: + pipeline-step: "run" + test-suite: ${{ matrix.type.test-suite }} + module-directory: ${{ matrix.type.module-directory }} + tag-filter: ${{ matrix.type.tag-filter }} + github-token: ${{ secrets.GITHUB_TOKEN }} + run-all-tests: "true" + run-concurrency: "32" + + - name: Update Test Index + uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison + with: + pipeline-step: "update" test-suite: ${{ matrix.type.test-suite }} module-directory: ${{ matrix.type.module-directory }} tag-filter: ${{ matrix.type.tag-filter }} - update-index: "true" - force-update-index: "true" - build-concurrency: "32" run-concurrency: "32" github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Print postgres logs if: ${{ always() }} From 4e632f00ccfacb0221a5cf2a0cb242a8080d3951 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Mon, 25 Nov 2024 14:43:58 -0800 Subject: [PATCH 3/9] feat: combine core unit tests --- .../actions/setup-ci-core-tests/action.yml | 24 +------------------ .github/workflows/ci-core-partial.yml | 22 ++++------------- 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/.github/actions/setup-ci-core-tests/action.yml b/.github/actions/setup-ci-core-tests/action.yml index c066c214042..cb74230f810 100644 --- a/.github/actions/setup-ci-core-tests/action.yml +++ b/.github/actions/setup-ci-core-tests/action.yml @@ -16,21 +16,9 @@ inputs: The expected database URL required: true - build-only: - description: | - Only setup the necessary dependencies for building the binary - default: "false" - runs: using: composite steps: - - name: Log Start - shell: bash - run: | - echo "====================================" - echo "Setting up CI Core Tests Environment" - echo "====================================" - - name: Setup NodeJS uses: ./.github/actions/setup-nodejs with: @@ -46,7 +34,7 @@ runs: - name: Replace chainlink-evm deps if: ${{ inputs.evm-ref-override != ''}} shell: bash - run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref }} + run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref-override }} - name: Setup Solana uses: ./.github/actions/setup-solana @@ -55,11 +43,9 @@ runs: uses: ./.github/actions/setup-wasmd - name: Setup Postgres - if: ${{ inputs.build-only == 'false' }} uses: ./.github/actions/setup-postgres - name: Touching core/web/assets/index.html - if: ${{ inputs.build-only == 'false' }} shell: bash run: mkdir -p core/web/assets && touch core/web/assets/index.html @@ -74,12 +60,10 @@ runs: run: go mod download - name: Build binary - if: ${{ inputs.build-only == 'false' }} shell: bash run: go build -o chainlink.test . - name: Setup DB - if: ${{ inputs.build-only == 'false' }} shell: bash run: ./chainlink.test local db preparetest env: @@ -101,9 +85,3 @@ runs: go install ./pkg/chainlink/cmd/chainlink-starknet popd - - name: Log End - shell: bash - run: | - echo "=============================================" - echo "Finished Setting up CI Core Tests Environment" - echo "=============================================" diff --git a/.github/workflows/ci-core-partial.yml b/.github/workflows/ci-core-partial.yml index 6da97d5c50a..1106d6d2e19 100644 --- a/.github/workflows/ci-core-partial.yml +++ b/.github/workflows/ci-core-partial.yml @@ -22,11 +22,8 @@ jobs: matrix: type: - test-suite: "core" - tag-filter: "" - module-directory: "./" - - test-suite: "integration" - tag-filter: "integration" module-directory: "./" + build-flags: "-tags=integration" - test-suite: "ccip-deployment" tag-filter: "" module-directory: "./deployment" @@ -38,11 +35,6 @@ jobs: - name: Checkout the repo uses: actions/checkout@v4.2.1 - - name: Change Modtime of Files (cache optimization) - shell: bash - run: | - find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true - - name: Setup CI Core Environment uses: ./.github/actions/setup-ci-core-tests with: @@ -53,11 +45,11 @@ jobs: uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison with: pipeline-step: "build" + build-concurrency: "32" test-suite: ${{ matrix.type.test-suite }} module-directory: ${{ matrix.type.module-directory }} - tag-filter: ${{ matrix.type.tag-filter }} - build-concurrency: "32" github-token: ${{ secrets.GITHUB_TOKEN }} + build-flags: ${{ matrix.type.build-flags }} - name: Run Tests uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison @@ -65,23 +57,17 @@ jobs: CL_DATABASE_URL: ${{ env.DB_URL }} with: pipeline-step: "run" + run-concurrency: "16" test-suite: ${{ matrix.type.test-suite }} module-directory: ${{ matrix.type.module-directory }} - tag-filter: ${{ matrix.type.tag-filter }} github-token: ${{ secrets.GITHUB_TOKEN }} - run-all-tests: "true" - run-concurrency: "32" - name: Update Test Index uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison with: pipeline-step: "update" test-suite: ${{ matrix.type.test-suite }} - module-directory: ${{ matrix.type.module-directory }} - tag-filter: ${{ matrix.type.tag-filter }} - run-concurrency: "32" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Print postgres logs if: ${{ always() }} From 473b11e56526f91591c4f1acd90246c89f34cd01 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Wed, 27 Nov 2024 11:28:33 -0800 Subject: [PATCH 4/9] update setup env action --- .../actions/setup-ci-core-tests/action.yml | 32 ++++++--- .github/workflows/ci-core-partial.yml | 10 +-- .github/workflows/ci-core.yml | 69 ++----------------- 3 files changed, 35 insertions(+), 76 deletions(-) diff --git a/.github/actions/setup-ci-core-tests/action.yml b/.github/actions/setup-ci-core-tests/action.yml index cb74230f810..378cc7477e7 100644 --- a/.github/actions/setup-ci-core-tests/action.yml +++ b/.github/actions/setup-ci-core-tests/action.yml @@ -1,11 +1,13 @@ name: Setup CI Core Tests description: Shared setup steps for ci-core inputs: - test-suite: - description: | - The test suite's name - required: true + go-mod-download-directory: + description: | + The directory to run go mod download in. If not provided, it will not run go mod download. + required: false + default: "" + evm-ref-override: description: | Overrides the evm/relayer dependency version @@ -16,6 +18,17 @@ inputs: The expected database URL required: true + build-cache-version: + description: | + The version of the build cache to use. Used for scoping build caches. + required: false + + restore-build-cache-only: + description: | + Whether to create a build cache + required: false + default: "false" + runs: using: composite steps: @@ -27,9 +40,8 @@ runs: - name: Setup Go uses: ./.github/actions/setup-go with: - # only restore for now - restore-build-cache-only: "true" - build-cache-version: ${{ inputs.test-suite }} + build-cache-version: ${{ inputs.build-cache-version }} + restore-build-cache-only: ${{ inputs.restore-build-cache-only }} - name: Replace chainlink-evm deps if: ${{ inputs.evm-ref-override != ''}} @@ -53,10 +65,10 @@ runs: shell: bash run: go mod download - - name: Go Mod Download (deployment) - if: ${{ matrix.type.test-suite == 'ccip-deployment' }} + - name: Go Mod Download (optional) + if: ${{ inputs.go-mod-download-directory != '' }} shell: bash - working-directory: "./deployment" + working-directory: ${{ inputs.go-mod-download-directory }} run: go mod download - name: Build binary diff --git a/.github/workflows/ci-core-partial.yml b/.github/workflows/ci-core-partial.yml index 1106d6d2e19..d41444fffac 100644 --- a/.github/workflows/ci-core-partial.yml +++ b/.github/workflows/ci-core-partial.yml @@ -25,7 +25,6 @@ jobs: module-directory: "./" build-flags: "-tags=integration" - test-suite: "ccip-deployment" - tag-filter: "" module-directory: "./deployment" env: # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests @@ -38,8 +37,10 @@ jobs: - name: Setup CI Core Environment uses: ./.github/actions/setup-ci-core-tests with: - test-suite: ${{ matrix.type.cmd }} + build-cache-version: ${{ matrix.type.test-suite }} + restore-build-cache-only: "true" db-url: ${{ env.DB_URL }} + go-mod-download-directory: ${{ matrix.type.test-suite == 'ccip-deployment' && matrix.type.module-directory || '' }} - name: Build Tests uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison @@ -66,6 +67,7 @@ jobs: uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison with: pipeline-step: "update" + force-update-index: "true" test-suite: ${{ matrix.type.test-suite }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -97,7 +99,7 @@ jobs: db-url: ${{ env.DB_URL }} - name: Increase Timeouts - if: ${{ github.event.schedule != ''}} + if: ${{ github.event_name == 'schedule'}} run: | echo "FUZZ_TIMEOUT_MINUTES=10">> $GITHUB_ENV @@ -134,7 +136,7 @@ jobs: db-url: ${{ env.DB_URL }} - name: Increase Timeouts - if: ${{ github.event.schedule != ''}} + if: ${{ github.event_name == 'schedule'}} run: | echo "TIMEOUT=10m" >> $GITHUB_ENV echo "COUNT=50" >> $GITHUB_ENV diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index c38ecd918ae..7ac82ab7908 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -154,74 +154,19 @@ jobs: uses: actions/checkout@v4.2.1 - name: Change Modtime of Files (cache optimization) + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} shell: bash run: | find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true - - - name: Setup NodeJS - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - uses: ./.github/actions/setup-nodejs - with: - prod: "true" - - - name: Setup Go + + - name: Setup CI Core Environment if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - uses: ./.github/actions/setup-go + uses: ./.github/actions/setup-ci-core-tests with: - # race/fuzz tests don't benefit repeated caching, so restore from develop's build cache - restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_fuzz' }} + db-url: ${{ env.DB_URL }} + evm-ref-override: ${{ github.event.inputs.evm-ref }} build-cache-version: ${{ matrix.type.cmd }} - - - name: Replace chainlink-evm deps - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' && inputs.evm-ref != ''}} - shell: bash - run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref }} - - - name: Setup Solana - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - uses: ./.github/actions/setup-solana - - - name: Setup wasmd - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - uses: ./.github/actions/setup-wasmd - - - name: Setup Postgres - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - uses: ./.github/actions/setup-postgres - - - name: Touching core/web/assets/index.html - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - run: mkdir -p core/web/assets && touch core/web/assets/index.html - - - name: Download Go vendor packages - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - run: go mod download - - - name: Build binary - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - run: go build -o chainlink.test . - - - name: Setup DB - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - run: ./chainlink.test local db preparetest - env: - CL_DATABASE_URL: ${{ env.DB_URL }} - - - name: Install LOOP Plugins - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - run: | - pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-feeds) - go install ./cmd/chainlink-feeds - popd - pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-data-streams) - go install ./mercury/cmd/chainlink-mercury - popd - pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-solana) - go install ./pkg/solana/cmd/chainlink-solana - popd - pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-starknet/relayer) - go install ./pkg/chainlink/cmd/chainlink-starknet - popd + restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_fuzz' }} - name: Increase Timeouts for Fuzz/Race # Increase timeouts for scheduled runs only From ebfab01faae55d189e085c94884dbd8014ef0fa8 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Wed, 27 Nov 2024 14:47:05 -0800 Subject: [PATCH 5/9] feat: collect coverage --- .github/workflows/ci-core-partial.yml | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/.github/workflows/ci-core-partial.yml b/.github/workflows/ci-core-partial.yml index d41444fffac..ba28d6360a7 100644 --- a/.github/workflows/ci-core-partial.yml +++ b/.github/workflows/ci-core-partial.yml @@ -47,6 +47,7 @@ jobs: with: pipeline-step: "build" build-concurrency: "32" + collect-coverage: "true" test-suite: ${{ matrix.type.test-suite }} module-directory: ${{ matrix.type.module-directory }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -59,6 +60,7 @@ jobs: with: pipeline-step: "run" run-concurrency: "16" + collect-coverage: "true" test-suite: ${{ matrix.type.test-suite }} module-directory: ${{ matrix.type.module-directory }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -75,6 +77,64 @@ jobs: if: ${{ always() }} run: docker compose logs postgres | tee ../../../postgres_logs.txt working-directory: ./.github/actions/setup-postgres + + scan: + name: SonarQube Scan + needs: run-unit-tests + runs-on: ubuntu-latest + steps: + - name: Checkout the repo + uses: actions/checkout@v4.2.1 + with: + # fetches all history for all tags and branches to provide more metadata for sonar reports + fetch-depth: 0 + + - name: Download all workflow run artifacts + uses: actions/download-artifact@v4.1.8 + with: + path: coverage + pattern: coverage-* + merge-multiple: true + + - name: Check and Set SonarQube Report Paths + shell: bash + run: | + ARGS="" + sonarqube_coverage_report_paths=$(find ./coverage -name '*.cover.out' | paste -sd "," -) + + # TODO uncomment when linting in enabled + # Check and assign paths for lint reports + # if [ -d "golangci-lint-report" ]; then + # sonarqube_lint_report_paths=$(find golangci-lint-report -name 'golangci-lint-report.xml' | paste -sd "," -) + # else + # sonarqube_lint_report_paths="" + # fi + # if [[ -z "$sonarqube_lint_report_paths" ]]; then + # echo "::warning::No lint report paths found, will not pass to sonarqube" + # else + # echo "Found lint report paths: $sonarqube_lint_report_paths" + # ARGS="$ARGS -Dsonar.go.golangci-lint.reportPaths=$sonarqube_lint_report_paths" + # fi + + if [[ -z "$sonarqube_coverage_report_paths" ]]; then + echo "::warning::No coverage report paths found, will not pass to sonarqube" + else + echo "Found coverage report paths: $sonarqube_coverage_report_paths" + ARGS="$ARGS -Dsonar.go.coverage.reportPaths=$sonarqube_coverage_report_paths" + fi + + echo "Final SONARQUBE_ARGS: $ARGS" + echo "SONARQUBE_ARGS=$ARGS" >> $GITHUB_ENV + + - name: SonarQube Scan + if: ${{ env.SONARQUBE_ARGS != '' }} + uses: sonarsource/sonarqube-scan-action@aecaf43ae57e412bd97d70ef9ce6076e672fe0a9 # v2.3.0 + with: + args: ${{ env.SONARQUBE_ARGS }} + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_SCANNER_OPTS: "-Xms6g -Xmx8g" run-fuzz-tests: From 6467b1929a42235a2dba91244cbdeba7a272c97e Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 28 Nov 2024 10:51:02 -0800 Subject: [PATCH 6/9] chore: cleanup --- .github/workflows/ci-core-partial.yml | 54 ++++++++++++++++++++------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-core-partial.yml b/.github/workflows/ci-core-partial.yml index ba28d6360a7..3c8f6e1236c 100644 --- a/.github/workflows/ci-core-partial.yml +++ b/.github/workflows/ci-core-partial.yml @@ -1,18 +1,40 @@ name: Core Unit Tests -# Run on key branches to make sure integration is good, otherwise run on all PR's on: push: branches: - develop - - main - "release/*" merge_group: pull_request: jobs: + filter: + name: Detect Changes + permissions: + pull-requests: read + outputs: + should-run-all-tests: ${{ steps.match-some.outputs.test-data == 'true' }} + should-collect-coverage: ${{ github.event_name == 'schedule' }} + runs-on: ubuntu-latest + steps: + - name: Checkout the repo + uses: actions/checkout@v4.2.1 + with: + repository: smartcontractkit/chainlink + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: match-some + with: + # "if any changed file matches one or more of the conditions" (https://github.com/dorny/paths-filter/issues/225) + predicate-quantifier: some + # test-data - any changes to any testdata files/paths + filters: | + test-data: + - '**/testdata/**' + run-unit-tests: name: Tests (${{ matrix.type.test-suite }}) + needs: filter runs-on: ubuntu22.04-32cores-128GB permissions: id-token: write @@ -43,33 +65,36 @@ jobs: go-mod-download-directory: ${{ matrix.type.test-suite == 'ccip-deployment' && matrix.type.module-directory || '' }} - name: Build Tests - uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison + uses: smartcontractkit/.github/apps/go-conditional-tests@feat/go-test-binary-comparison + timeout-minutes: 10 with: pipeline-step: "build" build-concurrency: "32" - collect-coverage: "true" + collect-coverage: ${{ needs.filter.outputs.should-collect-coverage }} test-suite: ${{ matrix.type.test-suite }} module-directory: ${{ matrix.type.module-directory }} github-token: ${{ secrets.GITHUB_TOKEN }} build-flags: ${{ matrix.type.build-flags }} - name: Run Tests - uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison + uses: smartcontractkit/.github/apps/go-conditional-tests@feat/go-test-binary-comparison + timeout-minutes: 15 env: CL_DATABASE_URL: ${{ env.DB_URL }} with: pipeline-step: "run" - run-concurrency: "16" - collect-coverage: "true" + run-concurrency: "24" + run-all-tests: ${{ needs.filter.outputs.should-run-all-tests }} + collect-coverage: ${{ needs.filter.outputs.should-collect-coverage }} test-suite: ${{ matrix.type.test-suite }} module-directory: ${{ matrix.type.module-directory }} github-token: ${{ secrets.GITHUB_TOKEN }} - name: Update Test Index - uses: smartcontractkit/.github/apps/go-test-caching@feat/go-test-binary-comparison + uses: smartcontractkit/.github/apps/go-conditional-tests@feat/go-test-binary-comparison with: pipeline-step: "update" - force-update-index: "true" + collect-coverage: ${{ needs.filter.outputs.should-collect-coverage }} test-suite: ${{ matrix.type.test-suite }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -80,7 +105,8 @@ jobs: scan: name: SonarQube Scan - needs: run-unit-tests + needs: [ run-unit-tests, filter ] + if: ${{ needs.filter.outputs.should-collect-coverage == 'true' }} runs-on: ubuntu-latest steps: - name: Checkout the repo @@ -139,7 +165,6 @@ jobs: run-fuzz-tests: name: Tests (fuzz) - if: ${{ github.ref == 'refs/heads/dont-run' }} runs-on: ubuntu22.04-32cores-128GB env: DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable @@ -155,7 +180,8 @@ jobs: - name: Setup CI Core Environment uses: ./.github/actions/setup-ci-core-tests with: - test-suite: "fuzz" + build-cache-version: "fuzz" + restore-build-cache-only: "true" db-url: ${{ env.DB_URL }} - name: Increase Timeouts @@ -176,7 +202,6 @@ jobs: run-race-tests: name: Tests (race) - if: ${{ github.ref == 'refs/heads/dont-run' }} runs-on: ubuntu22.04-32cores-128GB env: DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable @@ -192,7 +217,8 @@ jobs: - name: Setup CI Core Environment uses: ./.github/actions/setup-ci-core-tests with: - test-suite: "race" + build-cache-version: "race" + restore-build-cache-only: "true" db-url: ${{ env.DB_URL }} - name: Increase Timeouts From b8e686c5f505110cfb9db1a51e458016185f08b1 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Fri, 6 Dec 2024 13:23:48 -0800 Subject: [PATCH 7/9] chore: less run conditions, lower concurrency, proper action version --- .github/workflows/ci-core-partial.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-core-partial.yml b/.github/workflows/ci-core-partial.yml index 3c8f6e1236c..5548e08f35b 100644 --- a/.github/workflows/ci-core-partial.yml +++ b/.github/workflows/ci-core-partial.yml @@ -4,9 +4,12 @@ on: push: branches: - develop - - "release/*" - merge_group: + # - "release/*" + # merge_group: pull_request: + schedule: + - cron: "0 0 * * *" + jobs: filter: @@ -65,7 +68,7 @@ jobs: go-mod-download-directory: ${{ matrix.type.test-suite == 'ccip-deployment' && matrix.type.module-directory || '' }} - name: Build Tests - uses: smartcontractkit/.github/apps/go-conditional-tests@feat/go-test-binary-comparison + uses: smartcontractkit/.github/apps/go-conditional-tests@37882e110590e636627a26371bdbd56ddfcce821 # go-conditional-tests@0.1.0 timeout-minutes: 10 with: pipeline-step: "build" @@ -77,13 +80,13 @@ jobs: build-flags: ${{ matrix.type.build-flags }} - name: Run Tests - uses: smartcontractkit/.github/apps/go-conditional-tests@feat/go-test-binary-comparison + uses: smartcontractkit/.github/apps/go-conditional-tests@37882e110590e636627a26371bdbd56ddfcce821 # go-conditional-tests@0.1.0 timeout-minutes: 15 env: CL_DATABASE_URL: ${{ env.DB_URL }} with: pipeline-step: "run" - run-concurrency: "24" + run-concurrency: "16" run-all-tests: ${{ needs.filter.outputs.should-run-all-tests }} collect-coverage: ${{ needs.filter.outputs.should-collect-coverage }} test-suite: ${{ matrix.type.test-suite }} @@ -91,7 +94,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Update Test Index - uses: smartcontractkit/.github/apps/go-conditional-tests@feat/go-test-binary-comparison + uses: smartcontractkit/.github/apps/go-conditional-tests@37882e110590e636627a26371bdbd56ddfcce821 # go-conditional-tests@0.1.0 with: pipeline-step: "update" collect-coverage: ${{ needs.filter.outputs.should-collect-coverage }} From 460a75a60a18d4e7678436ecdbd1d3b3b305e629 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Fri, 6 Dec 2024 17:08:09 -0800 Subject: [PATCH 8/9] fix: dont modify ci-core.yml Did some testing trying to see if I could speed up test binary building through caching. It seems like the cache saving doesn't work when the composite action is nested. --- .github/workflows/ci-core.yml | 69 +++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 7ac82ab7908..c38ecd918ae 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -154,19 +154,74 @@ jobs: uses: actions/checkout@v4.2.1 - name: Change Modtime of Files (cache optimization) - if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} shell: bash run: | find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true - - - name: Setup CI Core Environment + + - name: Setup NodeJS if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} - uses: ./.github/actions/setup-ci-core-tests + uses: ./.github/actions/setup-nodejs with: - db-url: ${{ env.DB_URL }} - evm-ref-override: ${{ github.event.inputs.evm-ref }} - build-cache-version: ${{ matrix.type.cmd }} + prod: "true" + + - name: Setup Go + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} + uses: ./.github/actions/setup-go + with: + # race/fuzz tests don't benefit repeated caching, so restore from develop's build cache restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_fuzz' }} + build-cache-version: ${{ matrix.type.cmd }} + + - name: Replace chainlink-evm deps + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' && inputs.evm-ref != ''}} + shell: bash + run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref }} + + - name: Setup Solana + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} + uses: ./.github/actions/setup-solana + + - name: Setup wasmd + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} + uses: ./.github/actions/setup-wasmd + + - name: Setup Postgres + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} + uses: ./.github/actions/setup-postgres + + - name: Touching core/web/assets/index.html + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} + run: mkdir -p core/web/assets && touch core/web/assets/index.html + + - name: Download Go vendor packages + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} + run: go mod download + + - name: Build binary + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} + run: go build -o chainlink.test . + + - name: Setup DB + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} + run: ./chainlink.test local db preparetest + env: + CL_DATABASE_URL: ${{ env.DB_URL }} + + - name: Install LOOP Plugins + if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} + run: | + pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-feeds) + go install ./cmd/chainlink-feeds + popd + pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-data-streams) + go install ./mercury/cmd/chainlink-mercury + popd + pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-solana) + go install ./pkg/solana/cmd/chainlink-solana + popd + pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-starknet/relayer) + go install ./pkg/chainlink/cmd/chainlink-starknet + popd - name: Increase Timeouts for Fuzz/Race # Increase timeouts for scheduled runs only From 36bf9628b8d3d7b88105e5ec32cd581fe61cef2e Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Fri, 6 Dec 2024 17:16:06 -0800 Subject: [PATCH 9/9] fix: no nested composite actions --- .../actions/setup-ci-core-tests/action.yml | 46 +--------- .github/workflows/ci-core-partial.yml | 83 +++++++++++++------ 2 files changed, 61 insertions(+), 68 deletions(-) diff --git a/.github/actions/setup-ci-core-tests/action.yml b/.github/actions/setup-ci-core-tests/action.yml index 378cc7477e7..a2cf3f4103c 100644 --- a/.github/actions/setup-ci-core-tests/action.yml +++ b/.github/actions/setup-ci-core-tests/action.yml @@ -1,5 +1,8 @@ name: Setup CI Core Tests -description: Shared setup steps for ci-core +description: | + Shared setup steps for ci-core. + Note: Other actions should not be called from this action. There is + weird behavior when nesting reusable actions. inputs: go-mod-download-directory: @@ -7,56 +10,15 @@ inputs: The directory to run go mod download in. If not provided, it will not run go mod download. required: false default: "" - - evm-ref-override: - description: | - Overrides the evm/relayer dependency version - required: false db-url: description: | The expected database URL required: true - build-cache-version: - description: | - The version of the build cache to use. Used for scoping build caches. - required: false - - restore-build-cache-only: - description: | - Whether to create a build cache - required: false - default: "false" - runs: using: composite steps: - - name: Setup NodeJS - uses: ./.github/actions/setup-nodejs - with: - prod: "true" - - - name: Setup Go - uses: ./.github/actions/setup-go - with: - build-cache-version: ${{ inputs.build-cache-version }} - restore-build-cache-only: ${{ inputs.restore-build-cache-only }} - - - name: Replace chainlink-evm deps - if: ${{ inputs.evm-ref-override != ''}} - shell: bash - run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref-override }} - - - name: Setup Solana - uses: ./.github/actions/setup-solana - - - name: Setup wasmd - uses: ./.github/actions/setup-wasmd - - - name: Setup Postgres - uses: ./.github/actions/setup-postgres - - name: Touching core/web/assets/index.html shell: bash run: mkdir -p core/web/assets && touch core/web/assets/index.html diff --git a/.github/workflows/ci-core-partial.yml b/.github/workflows/ci-core-partial.yml index 5548e08f35b..c9752d4e1e4 100644 --- a/.github/workflows/ci-core-partial.yml +++ b/.github/workflows/ci-core-partial.yml @@ -8,8 +8,12 @@ on: # merge_group: pull_request: schedule: - - cron: "0 0 * * *" + - cron: "0 1 * * *" +env: + # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests + # when they should not be using it, while still allowing us to DRY up the setup + DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable jobs: filter: @@ -51,19 +55,33 @@ jobs: build-flags: "-tags=integration" - test-suite: "ccip-deployment" module-directory: "./deployment" - env: - # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests - # when they should not be using it, while still allowing us to DRY up the setup - DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable steps: - name: Checkout the repo uses: actions/checkout@v4.2.1 + - name: Setup NodeJS + uses: ./.github/actions/setup-nodejs + with: + prod: "true" + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + build-cache-version: ${{ matrix.type.test-suite }} + restore-build-cache-only: "false" + + - name: Setup Solana + uses: ./.github/actions/setup-solana + + - name: Setup wasmd + uses: ./.github/actions/setup-wasmd + + - name: Setup Postgres + uses: ./.github/actions/setup-postgres + - name: Setup CI Core Environment uses: ./.github/actions/setup-ci-core-tests with: - build-cache-version: ${{ matrix.type.test-suite }} - restore-build-cache-only: "true" db-url: ${{ env.DB_URL }} go-mod-download-directory: ${{ matrix.type.test-suite == 'ccip-deployment' && matrix.type.module-directory || '' }} @@ -108,7 +126,7 @@ jobs: scan: name: SonarQube Scan - needs: [ run-unit-tests, filter ] + needs: [run-unit-tests, filter] if: ${{ needs.filter.outputs.should-collect-coverage == 'true' }} runs-on: ubuntu-latest steps: @@ -116,7 +134,7 @@ jobs: uses: actions/checkout@v4.2.1 with: # fetches all history for all tags and branches to provide more metadata for sonar reports - fetch-depth: 0 + fetch-depth: 0 - name: Download all workflow run artifacts uses: actions/download-artifact@v4.1.8 @@ -164,27 +182,33 @@ jobs: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} SONAR_SCANNER_OPTS: "-Xms6g -Xmx8g" - run-fuzz-tests: name: Tests (fuzz) runs-on: ubuntu22.04-32cores-128GB - env: - DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable + if: ${{ github.event_name == 'schedule' }} steps: - name: Checkout the repo uses: actions/checkout@v4.2.1 - - name: Change Modtime of Files (cache optimization) - shell: bash - run: | - find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true + - name: Setup Go + uses: ./.github/actions/setup-go + with: + build-cache-version: "fuzz" + restore-build-cache-only: "true" + + - name: Setup Solana + uses: ./.github/actions/setup-solana + + - name: Setup wasmd + uses: ./.github/actions/setup-wasmd + + - name: Setup Postgres + uses: ./.github/actions/setup-postgres - name: Setup CI Core Environment uses: ./.github/actions/setup-ci-core-tests with: - build-cache-version: "fuzz" - restore-build-cache-only: "true" db-url: ${{ env.DB_URL }} - name: Increase Timeouts @@ -206,22 +230,29 @@ jobs: run-race-tests: name: Tests (race) runs-on: ubuntu22.04-32cores-128GB - env: - DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable + if: ${{ github.event_name == 'schedule' }} steps: - name: Checkout the repo uses: actions/checkout@v4.2.1 - - name: Change Modtime of Files (cache optimization) - shell: bash - run: | - find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true + - name: Setup Go + uses: ./.github/actions/setup-go + with: + build-cache-version: "race" + restore-build-cache-only: "true" + + - name: Setup Solana + uses: ./.github/actions/setup-solana + + - name: Setup wasmd + uses: ./.github/actions/setup-wasmd + + - name: Setup Postgres + uses: ./.github/actions/setup-postgres - name: Setup CI Core Environment uses: ./.github/actions/setup-ci-core-tests with: - build-cache-version: "race" - restore-build-cache-only: "true" db-url: ${{ env.DB_URL }} - name: Increase Timeouts