From 3aa8e2c76c10ea7de1de0dc0d2e89ffb78ee600c Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 31 Oct 2024 14:58:56 -0700 Subject: [PATCH] feat: optimize ci core build cache (#14988) * feat: optimize ci-core build cache * fix: shell script conditionals --- .github/actions/setup-go/action.yml | 26 +++- .github/workflows/ci-core.yml | 157 +++++++++++++++++------- tools/bin/go_core_ccip_deployment_tests | 25 ++-- tools/bin/go_core_fuzz | 15 +-- tools/bin/go_core_race_tests | 26 ++-- tools/bin/go_core_tests | 22 ++-- tools/bin/go_core_tests_integration | 52 ++++++++ 7 files changed, 226 insertions(+), 97 deletions(-) create mode 100755 tools/bin/go_core_tests_integration diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index 23af5dab70b..74897a4e9a0 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -7,6 +7,9 @@ inputs: cache-version: description: Set this to cache bust default: "1" + build-cache-version: + description: For scoping build caches to certain matrix jobs, takes precedence over cache-version + default: "" go-version-file: description: Set where the go version file is located at default: "go.mod" @@ -75,14 +78,31 @@ runs: restore-keys: | ${{ runner.os }}-gomod-${{ inputs.cache-version }}- + - uses: actions/cache/restore@v4.1.1 + name: Cache Go Build Outputs (restore) + # For certain events, we don't necessarily want to create a build cache, but we will benefit from restoring from one. + if: ${{ inputs.only-modules == 'false' && github.event == 'merge_queue' }} + with: + path: | + ${{ steps.go-cache-dir.outputs.gobuildcache }} + key: ${{ runner.os }}-gobuild-${{ inputs.build-cache-version || inputs.cache-version }}-${{ hashFiles(steps.go-module-path.outputs.path) }}-${{ steps.branch-name.outputs.current_branch }} + restore-keys: | + ${{ runner.os }}-gobuild-${{ inputs.build-cache-version || inputs.cache-version }}-${{ hashFiles(steps.go-module-path.outputs.path) }}-develop + ${{ runner.os }}-gobuild-${{ inputs.build-cache-version || inputs.cache-version }}-${{ hashFiles(steps.go-module-path.outputs.path) }}- + ${{ runner.os }}-gobuild-${{ inputs.build-cache-version || inputs.cache-version }}- + ${{ runner.os }}-gobuild-${{ inputs.cache-version }}- + - uses: actions/cache@v4.1.1 - if: ${{ inputs.only-modules == 'false' }} + # don't save cache on merge queue events, only restore + if: ${{ inputs.only-modules == 'false' && github.event != 'merge_queue' }} name: Cache Go Build Outputs with: path: | ${{ steps.go-cache-dir.outputs.gobuildcache }} # The lifetime of go build outputs is pretty short, so we make our primary cache key be the branch name - key: ${{ runner.os }}-gobuild-${{ inputs.cache-version }}-${{ hashFiles(steps.go-module-path.output.path) }}-${{ steps.branch-name.outputs.current_branch }} + key: ${{ runner.os }}-gobuild-${{ inputs.build-cache-version || inputs.cache-version }}-${{ hashFiles(steps.go-module-path.outputs.path) }}-${{ steps.branch-name.outputs.current_branch }} restore-keys: | - ${{ runner.os }}-gobuild-${{ inputs.cache-version }}-${{ hashFiles(steps.go-module-path.output.path) }}- + ${{ runner.os }}-gobuild-${{ inputs.build-cache-version || inputs.cache-version }}-${{ hashFiles(steps.go-module-path.outputs.path) }}-develop + ${{ runner.os }}-gobuild-${{ inputs.build-cache-version || inputs.cache-version }}-${{ hashFiles(steps.go-module-path.outputs.path) }}- + ${{ runner.os }}-gobuild-${{ inputs.build-cache-version || inputs.cache-version }}- ${{ runner.os }}-gobuild-${{ inputs.cache-version }}- diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 02c1359cdc2..862fefce320 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -58,7 +58,7 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' }} id: ignore-filter run: echo "changes=true" >> $GITHUB_OUTPUT - + golangci: # We don't directly merge dependabot PRs, so let's not waste the resources if: ${{ (github.event_name == 'pull_request' || github.event_name == 'schedule') && github.actor != 'dependabot[bot]' }} @@ -70,7 +70,7 @@ jobs: # For golangci-lint-action's `only-new-issues` option. pull-requests: read runs-on: ubuntu-24.04-8cores-32GB-ARM - needs: [filter] + needs: [filter, run-frequency] steps: - uses: actions/checkout@v4.2.1 - name: Golang Lint @@ -79,25 +79,8 @@ jobs: with: id: core name: lint - gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} - gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - - - name: Check Time For Slack Notify - if: ${{ failure() && github.event.schedule != ''}} - id: check-time - shell: bash - run: | - # Get the current hour (24-hour format) - current_hour=$(date +"%H") - - # Check if the current hour is 00 (midnight) - if [ "$current_hour" -eq "00" ]; then - echo "midnight=true" >> $GITHUB_OUTPUT - fi - - name: Notify Slack - if: ${{ failure() && github.event.schedule != '' && steps.check-time.outputs.midnight == 'true' }} + if: ${{ failure() && needs.run-frequency.outputs.one-per-day-frequency == 'true' }} uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 env: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} @@ -115,24 +98,23 @@ jobs: matrix: type: - cmd: go_core_tests - id: core_unit + os: ubuntu22.04-32cores-128GB + printResults: true + - cmd: go_core_tests_integration os: ubuntu22.04-32cores-128GB printResults: true - cmd: go_core_ccip_deployment_tests - id: core_unit os: ubuntu22.04-32cores-128GB printResults: true - cmd: go_core_race_tests - id: core_race - # use 64cores for overnight runs only due to massive number of runs from PRs - os: ${{ github.event_name == 'schedule' && 'ubuntu-latest-64cores-256GB' || 'ubuntu-latest-32cores-128GB' }} + # use 64cores for certain scheduled runs only + os: ${{ needs.run-frequency.outputs.two-per-day-frequency == 'true' && 'ubuntu-latest-64cores-256GB' || 'ubuntu-latest-32cores-128GB' }} - cmd: go_core_fuzz - id: core_fuzz os: ubuntu22.04-8cores-32GB name: Core Tests (${{ matrix.type.cmd }}) # We don't directly merge dependabot PRs, so let's not waste the resources - if: github.actor != 'dependabot[bot]' - needs: [filter] + if: ${{ github.actor != 'dependabot[bot]' }} + needs: [filter, run-frequency] runs-on: ${{ matrix.type.os }} permissions: id-token: write @@ -140,44 +122,61 @@ jobs: steps: - name: Checkout the repo uses: actions/checkout@v4.2.1 - - name: Setup node - if: ${{ needs.filter.outputs.changes == 'true' }} - uses: actions/setup-node@v4.0.4 + + - name: Change Modtime of Files (cache optimization) + shell: bash + run: | + find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true + - name: Setup NodeJS if: ${{ needs.filter.outputs.changes == 'true' }} uses: ./.github/actions/setup-nodejs with: prod: "true" + - name: Setup Go if: ${{ needs.filter.outputs.changes == 'true' }} uses: ./.github/actions/setup-go + with: + # race/fuzz tests don't benefit from build caching + only-modules: ${{ matrix.type.cmd == 'go_core_race_tests' || matrix.type.cmd == 'go_core_fuzz' }} + build-cache-version: ${{ matrix.type.cmd }} + - name: Replace chainlink-evm deps if: ${{ needs.filter.outputs.changes == 'true' && inputs.evm-ref != ''}} shell: bash run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref }} + - name: Setup Solana if: ${{ needs.filter.outputs.changes == 'true' }} uses: ./.github/actions/setup-solana + - name: Setup wasmd if: ${{ needs.filter.outputs.changes == 'true' }} uses: ./.github/actions/setup-wasmd + - name: Setup Postgres if: ${{ needs.filter.outputs.changes == 'true' }} uses: ./.github/actions/setup-postgres + - name: Touching core/web/assets/index.html if: ${{ needs.filter.outputs.changes == 'true' }} run: mkdir -p core/web/assets && touch core/web/assets/index.html + - name: Download Go vendor packages if: ${{ needs.filter.outputs.changes == 'true' }} run: go mod download + - name: Build binary if: ${{ needs.filter.outputs.changes == 'true' }} run: go build -o chainlink.test . + - name: Setup DB if: ${{ needs.filter.outputs.changes == 'true' }} run: ./chainlink.test local db preparetest env: CL_DATABASE_URL: ${{ env.DB_URL }} + - name: Install LOOP Plugins if: ${{ needs.filter.outputs.changes == 'true' }} run: | @@ -193,14 +192,17 @@ jobs: pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-starknet/relayer) go install ./pkg/chainlink/cmd/chainlink-starknet popd + - name: Increase Race Timeout + # Increase race timeout for scheduled runs only if: ${{ github.event.schedule != '' && needs.filter.outputs.changes == 'true' }} run: | echo "TIMEOUT=10m" >> $GITHUB_ENV echo "COUNT=50" >> $GITHUB_ENV + - name: Install gotestloghelper - if: ${{ needs.filter.outputs.changes == 'true' }} run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/gotestloghelper@v1.50.0 + - name: Run tests if: ${{ needs.filter.outputs.changes == 'true' }} id: run-tests @@ -208,13 +210,16 @@ jobs: OUTPUT_FILE: ./output.txt USE_TEE: false CL_DATABASE_URL: ${{ env.DB_URL }} + run: ./tools/bin/${{ matrix.type.cmd }} ./... + - name: Print Filtered Test Results if: ${{ failure() && needs.filter.outputs.changes == 'true' && steps.run-tests.conclusion == 'failure' }} run: | if [[ "${{ matrix.type.printResults }}" == "true" ]]; then cat output.txt | gotestloghelper -ci fi + - name: Print Races id: print-races if: ${{ failure() && matrix.type.cmd == 'go_core_race_tests' && needs.filter.outputs.changes == 'true' }} @@ -228,12 +233,14 @@ jobs: fi echo "github.event_name: ${{ github.event_name }}" echo "github.ref: ${{ github.ref }}" + - name: Print postgres logs if: ${{ always() && needs.filter.outputs.changes == 'true' }} run: docker compose logs postgres | tee ../../../postgres_logs.txt working-directory: ./.github/actions/setup-postgres + - name: Store logs artifacts - if: ${{ needs.filter.outputs.changes == 'true' && always() }} + if: ${{ always() && needs.filter.outputs.changes == 'true' }} uses: actions/upload-artifact@v4.4.3 with: name: ${{ matrix.type.cmd }}_logs @@ -244,6 +251,7 @@ jobs: ./coverage.txt ./postgres_logs.txt retention-days: 7 + - name: Notify Slack if: ${{ failure() && steps.print-races.outputs.post_to_slack == 'true' && matrix.type.cmd == 'go_core_race_tests' && (github.event_name == 'merge_group' || github.ref == 'refs/heads/develop') && needs.filter.outputs.changes == 'true' }} uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 @@ -252,16 +260,6 @@ jobs: with: channel-id: "#topic-data-races" slack-message: "Race tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" - - name: Collect Path Output - id: collect-path-output - env: - MATRIX_ID: ${{ matrix.type.id }} - run: | - # only push the test result file for the unit tests - if [[ "$MATRIX_ID" == "core_unit" ]]; then - resultsFile='{"testType":"go","filePath":"./output.txt"}' - echo "path_output=${resultsFile}" >> $GITHUB_OUTPUT - fi detect-flakey-tests: needs: [filter, core] @@ -276,40 +274,53 @@ jobs: steps: - name: Checkout the repo uses: actions/checkout@v4.2.1 + - name: Setup node uses: actions/setup-node@v4.0.4 + - name: Setup NodeJS uses: ./.github/actions/setup-nodejs with: prod: "true" + - name: Setup Go uses: ./.github/actions/setup-go + - name: Setup Postgres uses: ./.github/actions/setup-postgres + - name: Touching core/web/assets/index.html run: mkdir -p core/web/assets && touch core/web/assets/index.html + - name: Download Go vendor packages run: go mod download + - name: Replace chainlink-evm deps if: ${{ github.event_name == 'workflow_dispatch' && inputs.evm-ref != ''}} shell: bash run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref }} + - name: Build binary run: go build -o chainlink.test . + - name: Setup DB run: ./chainlink.test local db preparetest + - name: Load test outputs uses: actions/download-artifact@v4.1.8 with: name: go_core_tests_logs path: ./artifacts + - name: Delete go_core_tests_logs/coverage.txt shell: bash run: | # Need to delete coverage.txt so the disk doesn't fill up rm -f ./artifacts/go_core_tests_logs/coverage.txt + - name: Build flakey test runner run: go build ./tools/flakeytests/cmd/runner + - name: Re-run tests env: GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} @@ -340,21 +351,22 @@ jobs: scan: name: SonarQube Scan - needs: [core] - if: ${{ always() && github.actor != 'dependabot[bot]' }} + needs: [core, run-frequency] + if: ${{ always() && needs.run-frequency.outputs.four-per-day-frequency == 'true' && github.actor != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - name: Checkout the repo uses: actions/checkout@v4.2.1 with: fetch-depth: 0 # fetches all history for all tags and branches to provide more metadata for sonar reports + - name: Download all workflow run artifacts uses: actions/download-artifact@v4.1.8 - name: Check and Set SonarQube Report Paths shell: bash run: | - # Check and assign paths for coverage/test reports + # Check and assign paths for coverage/test reports in go_core_tests_logs if [ -d "go_core_tests_logs" ]; then sonarqube_coverage_report_paths=$(find go_core_tests_logs -name coverage.txt | paste -sd "," -) sonarqube_tests_report_paths=$(find go_core_tests_logs -name output.txt | paste -sd "," -) @@ -363,6 +375,15 @@ jobs: sonarqube_tests_report_paths="" fi + # Check and assign paths for coverage/test reports in go_core_tests_integration_logs + if [ -d "go_core_tests_integration_logs" ]; then + integration_coverage_paths=$(find go_core_tests_integration_logs -name coverage.txt | paste -sd "," -) + integration_tests_paths=$(find go_core_tests_integration_logs -name output.txt | paste -sd "," -) + # Append to existing paths if they are set, otherwise assign directly + sonarqube_coverage_report_paths="${sonarqube_coverage_report_paths:+$sonarqube_coverage_report_paths,}$integration_coverage_paths" + sonarqube_tests_report_paths="${sonarqube_tests_report_paths:+$sonarqube_tests_report_paths,}$integration_tests_paths" + fi + # Check and assign paths for lint reports if [ -d "golangci-lint-report" ]; then sonarqube_lint_report_paths=$(find golangci-lint-report -name golangci-lint-report.xml | paste -sd "," -) @@ -475,3 +496,47 @@ jobs: - run: make gomodtidy - name: Ensure clean after tidy run: git diff --minimal --exit-code + + run-frequency: + name: Scheduled Run Frequency + outputs: + one-per-day-frequency: ${{ steps.check-time.outputs.one-per-day-frequency || 'false' }} + two-per-day-frequency: ${{ steps.check-time.outputs.two-per-day-frequency || 'false' }} + four-per-day-frequency: ${{ steps.check-time.outputs.four-per-day-frequency || 'false' }} + six-per-day-frequency: ${{ steps.check-time.outputs.six-per-day-frequency || 'false' }} + runs-on: ubuntu-latest + steps: + - name: Check time and set frequencies + id: check-time + shell: bash + run: | + if [ "$GITHUB_EVENT_NAME" != "schedule" ]; then + # Not a scheduled event, set all frequencies to false + echo "one-per-day-frequency=false" >> $GITHUB_OUTPUT + echo "two-per-day-frequency=false" >> $GITHUB_OUTPUT + echo "four-per-day-frequency=false" >> $GITHUB_OUTPUT + echo "six-per-day-frequency=false" >> $GITHUB_OUTPUT + else + # Scheduled event, check current time for frequencies + current_hour=$(date +"%H") + + # Check if the current hour is 00 (one per day) + if [ "$current_hour" -eq "00" ]; then + echo "one-per-day-frequency=true" >> $GITHUB_OUTPUT + fi + + # Check if the current hour is 00 or 12 (twice per day) + if [ "$current_hour" -eq "00" ] || [ "$current_hour" -eq "12" ]; then + echo "two-per-day-frequency=true" >> $GITHUB_OUTPUT + fi + + # Check if the current hour is 00, 06, 12, or 18 (four times per day) + if [ "$current_hour" -eq "00" ] || [ "$current_hour" -eq "06" ] || [ "$current_hour" -eq "12" ] || [ "$current_hour" -eq "18" ]; then + echo "four-per-day-frequency=true" >> $GITHUB_OUTPUT + fi + + # Check if the current hour is one of 00, 04, 08, 12, 16, or 20 (six times per day) + if [ "$current_hour" -eq "00" ] || [ "$current_hour" -eq "04" ] || [ "$current_hour" -eq "08" ] || [ "$current_hour" -eq "12" ] || [ "$current_hour" -eq "16" ] || [ "$current_hour" -eq "20" ]; then + echo "six-per-day-frequency=true" >> $GITHUB_OUTPUT + fi + fi diff --git a/tools/bin/go_core_ccip_deployment_tests b/tools/bin/go_core_ccip_deployment_tests index 6ece5cfeac3..5249496cc0a 100755 --- a/tools/bin/go_core_ccip_deployment_tests +++ b/tools/bin/go_core_ccip_deployment_tests @@ -6,22 +6,23 @@ SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE="../output.txt" USE_TEE="${USE_TEE:-true}" -# To allow reuse in CI from other repositories -TOOLS_PATH=${TOOLS_PATH:-"./tools"} - +cd ./deployment || exit +go mod download echo "Failed tests and panics: ---------------------" echo "" -use_tee() { - if [ "$USE_TEE" = "true" ]; then - tee "$@" +if [[ $GITHUB_EVENT_NAME == "schedule" ]]; then + if [[ $DEBUG == "true" ]]; then + go test -json -vet=off ./... -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt | tee $OUTPUT_FILE else - cat > "$@" + go test -json -vet=off ./... -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt | cat > $OUTPUT_FILE fi -} - -cd ./deployment || exit -go mod download -go test -json ./... -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt | use_tee $OUTPUT_FILE +else + if [[ $DEBUG == "true" ]]; then + go test -vet=off ./... | tee $OUTPUT_FILE + else + go test -vet=off ./... | cat > $OUTPUT_FILE + fi +fi EXITCODE=${PIPESTATUS[0]} # Assert no known sensitive strings present in test logger output diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index d81e6909300..eb0334fe7ca 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -4,27 +4,14 @@ set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -USE_TEE="${USE_TEE:-true}" - -# To allow reuse in CI from other repositories -FUZZ_TOOL_PATH=${FUZZ_TOOL_PATH:-"./fuzz"} -GO_MODULE_ROOT_PATH=${GO_MODULE_ROOT_PATH:-"./"} FUZZ_TIMEOUT=${FUZZ_TIMEOUT:-10m} echo "Failed fuzz tests and panics: ---------------------" echo "" -use_tee() { - if [ "$USE_TEE" = "true" ]; then - tee "$@" - else - cat > "$@" - fi -} - # the amount of --seconds here is subject to change based on how long the CI job takes in the future # as we add more fuzz tests, we should take into consideration increasing this timelapse, so we can have enough coverage. # We are timing out after ~10mins in case the tests hang. (Current CI duration is ~8m, modify if needed) -timeout "${FUZZ_TIMEOUT}" "${FUZZ_TOOL_PATH}"/fuzz_all_native.py --ci --seconds 420 --go_module_root "${GO_MODULE_ROOT_PATH}" | use_tee $OUTPUT_FILE +timeout "${FUZZ_TIMEOUT}" ./fuzz/fuzz_all_native.py --ci --seconds 420 --go_module_root ./ | tee $OUTPUT_FILE EXITCODE=${PIPESTATUS[0]} # Assert no known sensitive strings present in test logger output diff --git a/tools/bin/go_core_race_tests b/tools/bin/go_core_race_tests index d0fcb6cae41..d09a8d903e4 100755 --- a/tools/bin/go_core_race_tests +++ b/tools/bin/go_core_race_tests @@ -1,23 +1,27 @@ #!/usr/bin/env bash set -ex OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -USE_TEE="${USE_TEE:-true}" TIMEOUT="${TIMEOUT:-30s}" COUNT="${COUNT:-10}" -# To allow reuse in CI from other repositories -TOOLS_PATH=${TOOLS_PATH:-"./tools"} - -GO_LDFLAGS=$(bash ${TOOLS_PATH}/bin/ldflags) -use_tee() { - if [ "$USE_TEE" = "true" ]; then - tee "$@" +echo "Failed tests and panics: ---------------------" +echo "" +if [[ $GITHUB_EVENT_NAME == "schedule" ]]; then + if [[ $DEBUG == "true" ]]; then + GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | tee $OUTPUT_FILE + else + GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE + fi +else + if [[ $DEBUG == "true" ]]; then + GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | tee $OUTPUT_FILE else - cat > "$@" + GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE fi -} -GORACE="log_path=$PWD/race" go test -json -race -ldflags "$GO_LDFLAGS" -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | use_tee "$OUTPUT_FILE" +fi EXITCODE=${PIPESTATUS[0]} + + # Fail if any race logs are present. if ls race.* &>/dev/null then diff --git a/tools/bin/go_core_tests b/tools/bin/go_core_tests index f7c1dfaf231..90713e15563 100755 --- a/tools/bin/go_core_tests +++ b/tools/bin/go_core_tests @@ -4,22 +4,22 @@ set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -USE_TEE="${USE_TEE:-true}" - -# To allow reuse in CI from other repositories -TOOLS_PATH=${TOOLS_PATH:-"./tools"} echo "Failed tests and panics: ---------------------" echo "" -GO_LDFLAGS=$(bash ${TOOLS_PATH}/bin/ldflags) -use_tee() { - if [ "$USE_TEE" = "true" ]; then - tee "$@" +if [[ $GITHUB_EVENT_NAME == "schedule" ]]; then + if [[ $DEBUG == "true" ]]; then + go test -json -vet=off -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt $1 | tee $OUTPUT_FILE else - cat > "$@" + go test -json -vet=off -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt $1 | cat > $OUTPUT_FILE fi -} -go test -json -ldflags "$GO_LDFLAGS" -tags integration $TEST_FLAGS -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt $1 | use_tee $OUTPUT_FILE +else + if [[ $DEBUG == "true" ]]; then + go test -vet=off $1 | tee $OUTPUT_FILE + else + go test -vet=off $1 | cat > $OUTPUT_FILE + fi +fi EXITCODE=${PIPESTATUS[0]} # Assert no known sensitive strings present in test logger output diff --git a/tools/bin/go_core_tests_integration b/tools/bin/go_core_tests_integration new file mode 100755 index 00000000000..6dfe22583cd --- /dev/null +++ b/tools/bin/go_core_tests_integration @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -o pipefail +set +e + +SCRIPT_PATH=$(dirname "$0"); SCRIPT_PATH=$(eval "cd \"$SCRIPT_PATH\" && pwd") +OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} + +echo "Finding and running integration-tagged tests" +INTEGRATION_TAGGED_TEST_FILES=$(find . -name '*_test.go' -exec grep -l '//go:build integration' {} +) +if [[ -z $INTEGRATION_TAGGED_TEST_FILES ]]; then + echo "No integration-tagged tests found." + exit 0 +fi +INTEGRATION_TEST_DIRS=$(echo "$INTEGRATION_TAGGED_TEST_FILES" | xargs -n1 dirname | sort -u) +INTEGRATION_TEST_DIRS_SPACE_DELIMITED=$(echo "$INTEGRATION_TEST_DIRS" | tr '\n' ' ') + +echo "Failed tests and panics: ---------------------" +echo "" +if [[ $GITHUB_EVENT_NAME == "schedule" ]]; then + # Experimental code to minimize size of this coverage report + # ALL_IMPORTS=$(go list -f '{{ join .Imports "\n" }}' $INTEGRATION_TEST_DIRS | sort -u) + # COVERPKG_DIRS=$(echo "$INTEGRATION_TEST_DIRS $ALL_IMPORTS" | grep "smartcontractkit/chainlink" | tr '\n' ',') + if [[ $DEBUG == "true" ]]; then + go test -json -tags integration -vet=off -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt $INTEGRATION_TEST_DIRS_SPACE_DELIMITED | tee $OUTPUT_FILE + else + go test -json -tags integration -vet=off -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt $INTEGRATION_TEST_DIRS_SPACE_DELIMITED | cat > $OUTPUT_FILE + fi +else + if [[ $DEBUG == "true" ]]; then + go test -vet=off -tags integration $INTEGRATION_TEST_DIRS_SPACE_DELIMITED | tee $OUTPUT_FILE + else + go test -vet=off -tags integration $INTEGRATION_TEST_DIRS_SPACE_DELIMITED | cat > $OUTPUT_FILE + fi +fi +EXITCODE=${PIPESTATUS[0]} + +# Assert no known sensitive strings present in test logger output +printf "\n----------------------------------------------\n\n" +echo "Beginning check of output logs for sensitive strings" +$SCRIPT_PATH/scrub_logs $OUTPUT_FILE +if [[ $? != 0 ]]; then + exit 1 +fi + +echo "Exit code: $EXITCODE" +if [[ $EXITCODE != 0 ]]; then + echo "Encountered test failures." +else + echo "All tests passed!" +fi +echo "go_core_tests exiting with code $EXITCODE" +exit $EXITCODE \ No newline at end of file