From dcca11893581aa9ae2440e08518b3bb68bf01e8b Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 18 Jan 2023 17:15:58 -0700 Subject: [PATCH] QA-134 Add Isomorphic Deployment ability to core smoke tests (#7951) * testing out new test deployment build * bump to latest chainlink-env isomorphic branch * switch to compilation in docker * network connection debugging * more debugging * additional ginkgo args * add sleep retry loop to test if first connection is just problematic * wrap test in if for local vs remote * merge conflict fixes and docker image updates * Merge conflict fixes * Fix integration-tests chainlink image version * add test image publish on merge to develop * Use existing version * Remove the need to add a test cleanup specifically for remote run tests * bump chainlink-env * Add fixes for soak testing * another clenv bump * fix for benchmark test * Update ocr2vrf types * Use chainlink-env v0.3.0 --- .../workflows/integration-tests-publish.yml | 43 ++++++ .github/workflows/integration-tests.yml | 46 +++++- .tool-versions | 3 +- CODEOWNERS | 1 + integration-tests/.tool-versions | 7 +- integration-tests/Makefile | 9 ++ integration-tests/README.md | 43 ++++-- integration-tests/actions/actions.go | 25 +--- .../benchmark/benchmark_runner_test.go | 2 + integration-tests/client/chainlink.go | 1 + integration-tests/go.mod | 4 +- integration-tests/go.sum | 8 +- integration-tests/known_networks.go | 3 +- integration-tests/scripts/buildTestImage | 41 ++++++ integration-tests/scripts/buildTests | 24 +++ integration-tests/scripts/entrypoint | 18 +++ integration-tests/scripts/run_product_tests | 1 - integration-tests/smoke/automation_test.go | 138 +++++++++++------- integration-tests/smoke/cron_test.go | 4 + integration-tests/smoke/flux_test.go | 4 + integration-tests/smoke/forwarder_ocr_test.go | 4 + integration-tests/smoke/keeper_test.go | 122 ++++++++++------ integration-tests/smoke/ocr2vrf_test.go | 4 + integration-tests/smoke/ocr_test.go | 6 + integration-tests/smoke/runlog_test.go | 4 + integration-tests/smoke/vrf_test.go | 4 + integration-tests/smoke/vrfv2_test.go | 4 + integration-tests/soak/soak_runner_test.go | 2 + integration-tests/test.Dockerfile | 10 ++ .../test.Dockerfile.dockerignore | 15 ++ 30 files changed, 456 insertions(+), 144 deletions(-) create mode 100644 .github/workflows/integration-tests-publish.yml create mode 100755 integration-tests/scripts/buildTestImage create mode 100755 integration-tests/scripts/buildTests create mode 100755 integration-tests/scripts/entrypoint create mode 100644 integration-tests/test.Dockerfile create mode 100644 integration-tests/test.Dockerfile.dockerignore diff --git a/.github/workflows/integration-tests-publish.yml b/.github/workflows/integration-tests-publish.yml new file mode 100644 index 00000000000..8707a45e209 --- /dev/null +++ b/.github/workflows/integration-tests-publish.yml @@ -0,0 +1,43 @@ +name: Integration Tests Publish +# Publish the compiled integration tests + +on: + push: + branches: + - develop + +env: + ECR_TAG: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:develop + +jobs: + publish-integration-test-image: + environment: integration + permissions: + id-token: write + contents: read + name: Publish Integration Test Image + runs-on: ubuntu-latest + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@v1 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: Publish Integration Test Image + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: build test runner + uses: smartcontractkit/chainlink-github-actions/docker/build-push@e72f0a768ac934afce498a802de893d89b12802f # v2.1.1 + with: + tags: ${{ env.ECR_TAG }} + file: ./integration-tests/test.Dockerfile + build-args: | + BASE_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/test-base-image + IMAGE_VERSION=v0.3.0 + SUITES="chaos migration performance reorg smoke soak benchmark" + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6e0398c5939..7bcbdd426c8 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,6 +7,10 @@ concurrency: group: integration-tests-chainlink-${{ github.ref }} cancel-in-progress: true +env: + CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:ci.${{ github.sha }} + jobs: changes: environment: integration @@ -35,6 +39,7 @@ jobs: continue-on-error: true outputs: src: ${{ steps.changes.outputs.src }} + build-chainlink: environment: integration permissions: @@ -67,6 +72,40 @@ jobs: this-job-name: Build Chainlink Image continue-on-error: true + build-test-image: + environment: integration + permissions: + id-token: write + contents: read + name: Build Test Image + runs-on: ubuntu20.04-16cores-64GB + needs: [changes] + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@v1 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: Build Test Image + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: build test runner + uses: smartcontractkit/chainlink-github-actions/docker/build-push@e72f0a768ac934afce498a802de893d89b12802f # v2.1.1 + with: + tags: ${{ env.ENV_JOB_IMAGE }} + file: ./integration-tests/test.Dockerfile + build-args: | + BASE_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/test-base-image + IMAGE_VERSION=v0.3.0 + SUITES="smoke" + #SUITES="chaos migration performance reorg smoke soak" + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + eth-smoke-tests-matrix: environment: integration permissions: @@ -75,7 +114,7 @@ jobs: id-token: write contents: read name: ETH Smoke Tests Matrix - needs: [build-chainlink, changes] + needs: [build-chainlink, changes, build-test-image] env: SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 CHAINLINK_COMMIT_SHA: ${{ github.sha }} @@ -126,6 +165,11 @@ jobs: - name: Run Tests if: needs.changes.outputs.src == 'true' uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e72f0a768ac934afce498a802de893d89b12802f # v2.1.1 + env: + TEST_SUITE: smoke + TEST_ARGS: -test.timeout 30m + CHAINLINK_IMAGE: ${{ env.CL_ECR }} + CHAINLINK_VERSION: latest.${{ github.sha }} with: test_command_to_run: make test_need_operator_assets && cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ./smoke/${{ matrix.product.name }}_test.go 2>&1 | tee /tmp/gotest.log | gotestfmt test_download_vendor_packages_command: make gomod diff --git a/.tool-versions b/.tool-versions index b1a045fcea0..cbc4920dcb4 100644 --- a/.tool-versions +++ b/.tool-versions @@ -2,7 +2,6 @@ golang 1.19.4 mockery 2.14.0 nodejs 16.13.2 postgres 13.3 -ginkgo 2.5.1 -helm 3.9.4 +helm 3.10.3 zig 0.10.0 golangci-lint 1.50.1 diff --git a/CODEOWNERS b/CODEOWNERS index ae6fb5fa687..1a2139b5e1a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -61,6 +61,7 @@ # CI/CD /.github/** @alexroan @chainchad @javuto @jkongie @jmank88 @samsondav /.github/workflows/integration-tests.yml @kalverra @tateexon @skudasov @anieeg +/.github/workflows/integration-tests-publish.yml @kalverra @tateexon @skudasov @anieeg /.github/workflows/integration-chaos-tests.yml @kalverra @tateexon @skudasov @anieeg /.github/workflows/performance-tests.yml @kalverra @tateexon @skudasov @anieeg /core/chainlink.Dockerfile @smartcontractkit/prodsec-public diff --git a/integration-tests/.tool-versions b/integration-tests/.tool-versions index 1db4855e835..271c4cb9fd5 100644 --- a/integration-tests/.tool-versions +++ b/integration-tests/.tool-versions @@ -1,3 +1,4 @@ -golang 1.19.4 -k3d 5.4.4 -kubectl 1.25.3 +golang 1.19.5 +k3d 5.4.6 +kubectl 1.25.5 +nodejs 18.13.0 diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 3d2cd0ee600..52a4856730b 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -55,6 +55,15 @@ install_gotestfmt: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest set -euo pipefail +# Builds the test image +# tag: the tag for the test image being built, example: tag=tate +# base_tag: the tag for the base-test-image to use, example: base_tag=latest +# suite: the test suites to build into the image, example: suite="chaos soak smoke reorg migration performance" +# push: set to true if you want the image pushed or leave blank if not, example: push=true +.PHONY: build_test_image +build_test_image: + ./scripts/buildTestImage $(tag) $(base_tag) "$(suite)" $(push) + ## All commands will use 16 threads to run tests in parallel. To change this, use -test.parallel n # Smoke diff --git a/integration-tests/README.md b/integration-tests/README.md index 0da34a0d84a..fd6a8ba1598 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -14,14 +14,6 @@ Prerequisites to run the tests. [Install](https://go.dev/doc/install) -
- Install Ginkgo - - [Ginkgo](https://onsi.github.io/ginkgo/) is the testing framework we use to compile and run our tests. It comes with a lot of handy testing setups and goodies on top of the standard Go testing packages. - - `go install github.com/onsi/ginkgo/v2/ginkgo` -
-
Install NodeJS @@ -76,17 +68,40 @@ make test_smoke_simulated # Run all smoke tests on a simulated network Run all smoke tests in parallel, only using simulated blockchains. *Note: As of now, you can only run tests in parallel on simulated chains, not on live ones. Running on parallel tests on live chains will give errors* ```sh -make test_smoke_simulated args="-nodes=" +make test_smoke_simulated args="-test.parallel=" ``` -You can also run specific tests using `make test_smoke` and a `focus` tag. +You can also run specific tests and debug tests in vscode by setting up your .vscode/settings.json with this information. Just replace all the "" with your information before running a test. + +```json +{ + "makefile.extensionOutputFolder": "./.vscode", + "go.testEnvVars": { + "LOG_LEVEL": "debug", + "SELECTED_NETWORKS": "SIMULATED,SIMULATED_1,SIMULATED_2", + "CHAINLINK_IMAGE":".dkr.ecr.us-west-2.amazonaws.com/chainlink", + "CHAINLINK_VERSION":"develop", + "CHAINLINK_ENV_USER":"", + "TEST_LOG_LEVEL":"debug", + "AWS_ACCESS_KEY_ID":"", + "AWS_SECRET_ACCESS_KEY":"", + "AWS_SESSION_TOKEN":"" + }, + "go.testTimeout": "900s" +} +``` +You can also run your tests inside of kubernetes instead of from locally to reduce local resource usage and the number of ports that get forwarded to the cluster. This is not recommended for normal developement since building and pushing the image can be time heavy depending on your internet upload speeds. To do this you will want to either pull down an already built chainlink-tests image or build one yourself. To build and push one yourself you can run: ```sh -make test_smoke args="-focus=@ocr" # Runs all the ocr smoke tests -make test_smoke args="-focus=@keeper" # Runs all smoke tests for keepers +make build_test_image tag= base_tag=latest suite="smoke soak chaos reorg migration performance" push=true ``` - -[Check out](https://onsi.github.io/ginkgo/#description-based-filtering) how Ginkgo handles focus and skip tags if you're looking for more precise behavior. +Once that is done building you can add this to your go.testEnvVars in .vscode/settings.json with the correct account number and tag filled out. +```json + "TEST_SUITE": "smoke", + "TEST_ARGS": "-test.timeout 30m", + "ENV_JOB_IMAGE":".dkr.ecr.us-west-2.amazonaws.com/chainlink-env-tests:", +``` +Once that is done you can run/debug your test using the vscode test view just like normal. ### Soak diff --git a/integration-tests/actions/actions.go b/integration-tests/actions/actions.go index 8766e6dc575..f656fea3d65 100644 --- a/integration-tests/actions/actions.go +++ b/integration-tests/actions/actions.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "math/big" - "os" "strings" "testing" @@ -230,21 +229,18 @@ func TeardownSuite( optionalTestReporter testreporters.TestReporter, // Optionally pass in a test reporter to log further metrics clients ...blockchain.EVMClient, ) error { - keepEnvs := os.Getenv("KEEP_ENVIRONMENTS") - if keepEnvs == "" { - keepEnvs = "NEVER" - } - if err := testreporters.WriteTeardownLogs(t, env, optionalTestReporter); err != nil { return errors.Wrap(err, "Error dumping environment logs, leaving environment running for manual retrieval") } for _, c := range clients { if c != nil && chainlinkNodes != nil && len(chainlinkNodes) > 0 { if err := returnFunds(chainlinkNodes, c); err != nil { + // This printed line is required for tests that use real funds to propagate the failure + // out to the system running the test. Do not remove + fmt.Println(environment.FAILED_FUND_RETURN) log.Error().Err(err).Str("Namespace", env.Cfg.Namespace). Msg("Error attempting to return funds from chainlink nodes to network's default wallet. " + "Environment is left running so you can try manually!") - keepEnvs = "ALWAYS" } } else { log.Info().Msg("Successfully returned funds from chainlink nodes to default network wallets") @@ -258,20 +254,7 @@ func TeardownSuite( } } - switch strings.ToUpper(keepEnvs) { - case "ALWAYS": - return nil - case "ONFAIL": - if !t.Failed() { - return env.Shutdown() - } - case "NEVER": - return env.Shutdown() - default: - log.Warn().Str("Invalid Keep Value", keepEnvs). - Msg("Invalid 'keep_environments' value, see the KEEP_ENVIRONMENTS env var") - } - return nil + return env.Shutdown() } // TeardownRemoteSuite is used when running a test within a remote-test-runner, like for long-running performance and diff --git a/integration-tests/benchmark/benchmark_runner_test.go b/integration-tests/benchmark/benchmark_runner_test.go index d6576d8acf9..e23be274008 100644 --- a/integration-tests/benchmark/benchmark_runner_test.go +++ b/integration-tests/benchmark/benchmark_runner_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/rs/zerolog/log" + "github.com/smartcontractkit/chainlink-env/config" "github.com/smartcontractkit/chainlink-env/pkg/cdk8s/blockscout" "github.com/stretchr/testify/require" @@ -183,6 +184,7 @@ func benchmarkTestHelper( for key, value := range activeEVMNetwork.ToMap() { remoteRunnerValues[key] = value } + remoteRunnerValues[config.EnvVarInsideK8s] = "true" remoteRunnerWrapper := map[string]interface{}{ "remote_test_runner": remoteRunnerValues, } diff --git a/integration-tests/client/chainlink.go b/integration-tests/client/chainlink.go index 0b1600a64b7..674cdf06943 100644 --- a/integration-tests/client/chainlink.go +++ b/integration-tests/client/chainlink.go @@ -46,6 +46,7 @@ func NewChainlink(c *ChainlinkConfig) (*Chainlink, error) { session := &Session{Email: c.Email, Password: c.Password} resp, err := rc.R().SetBody(session).Post("/sessions") if err != nil { + log.Info().Interface("session", session).Msg("session used") return nil, err } rc.SetCookies(resp.Cookies()) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index e192fee142a..2d86df8e72d 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -12,11 +12,11 @@ require ( github.com/satori/go.uuid v1.2.0 github.com/slack-go/slack v0.11.4 github.com/smartcontractkit/chainlink v1.10.0 - github.com/smartcontractkit/chainlink-env v0.2.60 + github.com/smartcontractkit/chainlink-env v0.3.0 github.com/smartcontractkit/chainlink-testing-framework v1.9.0 github.com/smartcontractkit/libocr v0.0.0-20221209172631-568a30f68407 github.com/smartcontractkit/ocr2keepers v0.6.6 - github.com/smartcontractkit/ocr2vrf v0.0.0-20230117141905-c3a5c0467fd4 + github.com/smartcontractkit/ocr2vrf v0.0.0-20230117145849-09a91a822ee2 github.com/stretchr/testify v1.8.1 github.com/umbracle/ethgo v0.1.3 go.dedis.ch/kyber/v3 v3.0.14 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index c7dfbbb8d39..fe6182fe134 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1599,8 +1599,8 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/slack-go/slack v0.11.4 h1:ojSa7KlPm3PqY2AomX4VTxEsK5eci5JaxCjlzGV5zoM= github.com/slack-go/slack v0.11.4/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= -github.com/smartcontractkit/chainlink-env v0.2.60 h1:Vu7GBh9Z25pWR9kyZacBUfu7MHUyYvUsK5ub77MVuco= -github.com/smartcontractkit/chainlink-env v0.2.60/go.mod h1:zspYFdXS57CzKkpcLkFEPuMQHOmI3L9H5idoPxobxRM= +github.com/smartcontractkit/chainlink-env v0.3.0 h1:HfyjiXeWUEDTI5vM7asSpTzqrx/9R71915MNEta+Ihw= +github.com/smartcontractkit/chainlink-env v0.3.0/go.mod h1:zspYFdXS57CzKkpcLkFEPuMQHOmI3L9H5idoPxobxRM= github.com/smartcontractkit/chainlink-relay v0.1.6-0.20221025223751-9b407cff57eb h1:NF6//JILgK8AeLkknJFEVsVRt+VqwNnxJ4SLpHKje9c= github.com/smartcontractkit/chainlink-relay v0.1.6-0.20221025223751-9b407cff57eb/go.mod h1:v/QSrVm3z4/aPz/PLB6da05B/r4MHZy0/jder7iPxkQ= github.com/smartcontractkit/chainlink-solana v1.0.2-0.20220930034647-edd5a863b876 h1:uctLwzPqXUbWWcOiZaltKNtb2XfIDVE1yQ04uLZ3N7Q= @@ -1613,8 +1613,8 @@ github.com/smartcontractkit/libocr v0.0.0-20221209172631-568a30f68407 h1:P3dhh6U github.com/smartcontractkit/libocr v0.0.0-20221209172631-568a30f68407/go.mod h1:5JnCHuYgmIP9ZyXzgAfI5Iwu0WxBtBKp+ApeT5o1Cjw= github.com/smartcontractkit/ocr2keepers v0.6.6 h1:21wVK5HGHLcfHo4HU4QW8wLCvhENFMhkXdqxMcHPgkU= github.com/smartcontractkit/ocr2keepers v0.6.6/go.mod h1:0y0wgX0OQTyOb04f823ifLvzsca+rsyE3ngGnmG8zxU= -github.com/smartcontractkit/ocr2vrf v0.0.0-20230117141905-c3a5c0467fd4 h1:ZS37mHqFOpbVj45sJ3bJQ2UJzRxhBkxFflyPNCT5z4A= -github.com/smartcontractkit/ocr2vrf v0.0.0-20230117141905-c3a5c0467fd4/go.mod h1:NKkp8yf3trq+hJe/gn2Q1dd4abZvcYavUKPzYJCgVew= +github.com/smartcontractkit/ocr2vrf v0.0.0-20230117145849-09a91a822ee2 h1:xiaI621shLKa4oK81bMMZdQZq5s08iWSNzWRM7LaqoA= +github.com/smartcontractkit/ocr2vrf v0.0.0-20230117145849-09a91a822ee2/go.mod h1:NKkp8yf3trq+hJe/gn2Q1dd4abZvcYavUKPzYJCgVew= github.com/smartcontractkit/sqlx v1.3.5-0.20210805004948-4be295aacbeb h1:OMaBUb4X9IFPLbGbCHsMU+kw/BPCrewaVwWGIBc0I4A= github.com/smartcontractkit/sqlx v1.3.5-0.20210805004948-4be295aacbeb/go.mod h1:HNUu4cJekUdsJbwRBCiOybtkPJEfGRELQPe2tkoDEyk= github.com/smartcontractkit/terra.go v1.0.3-0.20220108002221-62b39252ee16 h1:k+E0RKzVSG1QpxXakNUtcGUhq4ZMe0MAJ5Awg/l9oSc= diff --git a/integration-tests/known_networks.go b/integration-tests/known_networks.go index 969ebc2beee..e741c6ec861 100644 --- a/integration-tests/known_networks.go +++ b/integration-tests/known_networks.go @@ -179,7 +179,8 @@ var ( func determineSelectedNetworks() []blockchain.EVMNetwork { logging.Init() selectedNetworks := make([]blockchain.EVMNetwork, 0) - setNetworkNames := strings.Split(strings.ToUpper(os.Getenv("SELECTED_NETWORKS")), ",") + rawSelectedNetworks := strings.ToUpper(os.Getenv("SELECTED_NETWORKS")) + setNetworkNames := strings.Split(rawSelectedNetworks, ",") for _, setNetworkName := range setNetworkNames { if chosenNetwork, valid := mappedNetworks[setNetworkName]; valid { diff --git a/integration-tests/scripts/buildTestImage b/integration-tests/scripts/buildTestImage new file mode 100755 index 00000000000..ca2b4bb139e --- /dev/null +++ b/integration-tests/scripts/buildTestImage @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# build test binaries +# accepts a single space separated argument of the folders to build + +set -ex + +# get this scripts directory +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +cd "$SCRIPT_DIR"/../../ || exit 1 + +TAG_VERSION="${1}" +BASE_IMAGE_VERSION="${2}" +SUITES=$3 +DEFAULT_SUITES="smoke soak chaos" +ACCOUNT=$(aws sts get-caller-identity | jq -r .Account) +AWS_BASE="${ACCOUNT}".dkr.ecr.us-west-2.amazonaws.com +TAG="${AWS_BASE}"/chainlink-tests:"${TAG_VERSION}" +BASE_IMAGE="${AWS_BASE}"/test-base-image + +if [ "${TAG_VERSION}" = "" ]; then + echo "Need an argument for the image tag version in argument 1" + exit 1 +fi + +if [ "${BASE_IMAGE_VERSION}" = "" ]; then + echo "Need an argument for the test-base-image tag version in argument 2" + exit 1 +fi + +if [ "${SUITES}" = "" ]; then + echo "SUITES not set, using defaults \"${DEFAULT_SUITES}\"" + SUITES=${DEFAULT_SUITES} +fi + +aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin +docker build -t "${TAG}" -f "./integration-tests/test.Dockerfile" --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg IMAGE_VERSION="${BASE_IMAGE_VERSION}" --build-arg SUITES="${SUITES}" . +if "${4}" = "true"]; then + docker push "${TAG}" +fi diff --git a/integration-tests/scripts/buildTests b/integration-tests/scripts/buildTests new file mode 100755 index 00000000000..c62985a689a --- /dev/null +++ b/integration-tests/scripts/buildTests @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# build test binaries +# accepts a single space separated argument of the folders to build + +set -ex + +# get this scripts directory +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +cd "$SCRIPT_DIR"/../ || exit 1 + +# parse out quotes if they exist in the string +temp="${1%\"}" +tosplit="${temp#\"}" + +# find the suite name +OIFS=$IFS +IFS=' ' +for x in $tosplit +do + go test -c ./"${x}" +done +IFS=$OIFS diff --git a/integration-tests/scripts/entrypoint b/integration-tests/scripts/entrypoint new file mode 100755 index 00000000000..156b20d3496 --- /dev/null +++ b/integration-tests/scripts/entrypoint @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Runs tests for a specific product + +set -ex + +# get this scripts directory +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +cd "$SCRIPT_DIR"/../ || exit 1 + +# Arguments needed +# ARGS=${ARGS:=} any extra args for go test +# SUITE=${SUITE:=} the suite of tests you want to run +# TEST_NAME=${TEST_NAME:=} The specific test to run + +# run the tests +./${SUITE}.test -test.v -test.count 1 ${ARGS} -test.run ^${TEST_NAME}$ diff --git a/integration-tests/scripts/run_product_tests b/integration-tests/scripts/run_product_tests index b0513aac012..48899d45cfe 100755 --- a/integration-tests/scripts/run_product_tests +++ b/integration-tests/scripts/run_product_tests @@ -11,4 +11,3 @@ cd "$SCRIPT_DIR"/../ || exit 1 PRODUCT=${PRODUCT:=cron} ARGS=${ARGS:=} go test -v ${ARGS} ./smoke/"$PRODUCT"_test.go - \ No newline at end of file diff --git a/integration-tests/smoke/automation_test.go b/integration-tests/smoke/automation_test.go index e3aff426f33..61d7e1c3ef1 100644 --- a/integration-tests/smoke/automation_test.go +++ b/integration-tests/smoke/automation_test.go @@ -97,9 +97,12 @@ func TestMain(m *testing.M) { func TestAutomatedBasic(t *testing.T) { t.Parallel() - chainClient, _, contractDeployer, linkToken, registry, registrar := setupAutomationTest( + chainClient, _, contractDeployer, linkToken, registry, registrar, onlyStartRunner := setupAutomationTest( t, "basic-upkeep", ethereum.RegistryVersion_2_0, defaultOCRRegistryConfig, ) + if onlyStartRunner { + return + } consumers, upkeepIDs := actions.DeployConsumers( t, @@ -160,9 +163,12 @@ func TestAutomatedBasic(t *testing.T) { func TestAutomatedAddFunds(t *testing.T) { t.Parallel() - chainClient, _, contractDeployer, linkToken, registry, registrar := setupAutomationTest( + chainClient, _, contractDeployer, linkToken, registry, registrar, onlyStartRunner := setupAutomationTest( t, "add-funds", ethereum.RegistryVersion_2_0, defaultOCRRegistryConfig, ) + if onlyStartRunner { + return + } consumers, upkeepIDs := actions.DeployConsumers( t, @@ -208,9 +214,12 @@ func TestAutomatedAddFunds(t *testing.T) { func TestAutomatedPauseUnPause(t *testing.T) { t.Parallel() - chainClient, _, contractDeployer, linkToken, registry, registrar := setupAutomationTest( + chainClient, _, contractDeployer, linkToken, registry, registrar, onlyStartRunner := setupAutomationTest( t, "pause-unpause", ethereum.RegistryVersion_2_0, defaultOCRRegistryConfig, ) + if onlyStartRunner { + return + } consumers, upkeepIDs := actions.DeployConsumers( t, @@ -288,9 +297,12 @@ func TestAutomatedPauseUnPause(t *testing.T) { func TestAutomatedRegisterUpkeep(t *testing.T) { t.Parallel() - chainClient, _, contractDeployer, linkToken, registry, registrar := setupAutomationTest( + chainClient, _, contractDeployer, linkToken, registry, registrar, onlyStartRunner := setupAutomationTest( t, "register-upkeep", ethereum.RegistryVersion_2_0, defaultOCRRegistryConfig, ) + if onlyStartRunner { + return + } consumers, upkeepIDs := actions.DeployConsumers( t, @@ -357,9 +369,12 @@ func TestAutomatedRegisterUpkeep(t *testing.T) { func TestAutomatedPauseRegistry(t *testing.T) { t.Parallel() - chainClient, _, contractDeployer, linkToken, registry, registrar := setupAutomationTest( + chainClient, _, contractDeployer, linkToken, registry, registrar, onlyStartRunner := setupAutomationTest( t, "pause-registry", ethereum.RegistryVersion_2_0, defaultOCRRegistryConfig, ) + if onlyStartRunner { + return + } consumers, upkeepIDs := actions.DeployConsumers( t, @@ -412,9 +427,12 @@ func TestAutomatedPauseRegistry(t *testing.T) { func TestAutomatedKeeperNodesDown(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken, registry, registrar := setupAutomationTest( + chainClient, chainlinkNodes, contractDeployer, linkToken, registry, registrar, onlyStartRunner := setupAutomationTest( t, "keeper-nodes-down", ethereum.RegistryVersion_2_0, defaultOCRRegistryConfig, ) + if onlyStartRunner { + return + } consumers, upkeepIDs := actions.DeployConsumers( t, @@ -495,9 +513,12 @@ func TestAutomatedKeeperNodesDown(t *testing.T) { func TestAutomatedPerformSimulation(t *testing.T) { t.Parallel() - chainClient, _, contractDeployer, linkToken, registry, registrar := setupAutomationTest( + chainClient, _, contractDeployer, linkToken, registry, registrar, onlyStartRunner := setupAutomationTest( t, "perform-simulation", ethereum.RegistryVersion_2_0, defaultOCRRegistryConfig, ) + if onlyStartRunner { + return + } consumersPerformance, _ := actions.DeployPerformanceConsumers( t, @@ -546,9 +567,12 @@ func TestAutomatedPerformSimulation(t *testing.T) { func TestAutomatedCheckPerformGasLimit(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken, registry, registrar := setupAutomationTest( + chainClient, chainlinkNodes, contractDeployer, linkToken, registry, registrar, onlyStartRunner := setupAutomationTest( t, "gas-limit", ethereum.RegistryVersion_2_0, defaultOCRRegistryConfig, ) + if onlyStartRunner { + return + } consumersPerformance, upkeepIDs := actions.DeployPerformanceConsumers( t, @@ -643,9 +667,12 @@ func TestAutomatedCheckPerformGasLimit(t *testing.T) { func TestUpdateCheckData(t *testing.T) { t.Parallel() - chainClient, _, contractDeployer, linkToken, registry, registrar := setupAutomationTest( + chainClient, _, contractDeployer, linkToken, registry, registrar, onlyStartRunner := setupAutomationTest( t, "update-check-data", ethereum.RegistryVersion_2_0, defaultOCRRegistryConfig, ) + if onlyStartRunner { + return + } performDataChecker, upkeepIDs := actions.DeployPerformDataCheckerConsumers( t, @@ -707,12 +734,13 @@ func setupAutomationTest( registryVersion ethereum.KeeperRegistryVersion, registryConfig contracts.KeeperRegistrySettings, ) ( - blockchain.EVMClient, - []*client.Chainlink, - contracts.ContractDeployer, - contracts.LinkToken, - contracts.KeeperRegistry, - contracts.KeeperRegistrar, + chainClient blockchain.EVMClient, + chainlinkNodes []*client.Chainlink, + contractDeployer contracts.ContractDeployer, + linkToken contracts.LinkToken, + registry contracts.KeeperRegistry, + registrar contracts.KeeperRegistrar, + onlyStartRunner bool, ) { network := networks.SelectedNetwork evmConfig := eth.New(nil) @@ -726,6 +754,7 @@ func setupAutomationTest( testEnvironment := environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-automation-%s-%s", testName, strings.ReplaceAll(strings.ToLower(network.Name), " ", "-")), + Test: t, }). AddHelm(mockservercfg.New(nil)). AddHelm(mockserver.New(nil)). @@ -735,46 +764,49 @@ func setupAutomationTest( "toml": client.AddNetworksConfig(automationBaseTOML, network), })) err := testEnvironment.Run() + require.NoError(t, err, "Error setting up test environment") - chainClient, err := blockchain.NewEVMClient(network, testEnvironment) - require.NoError(t, err, "Error connecting to blockchain") - contractDeployer, err := contracts.NewContractDeployer(chainClient) - require.NoError(t, err, "Error building contract deployer") - chainlinkNodes, err := client.ConnectChainlinkNodes(testEnvironment) - require.NoError(t, err, "Error connecting to Chainlink nodes") - chainClient.ParallelTransactions(true) - - // Register cleanup for any test - t.Cleanup(func() { - err := actions.TeardownSuite(t, testEnvironment, utils.ProjectRoot, chainlinkNodes, nil, chainClient) - require.NoError(t, err, "Error tearing down environment") - }) - - txCost, err := chainClient.EstimateCostForChainlinkOperations(1000) - require.NoError(t, err, "Error estimating cost for Chainlink Operations") - err = actions.FundChainlinkNodes(chainlinkNodes, chainClient, txCost) - require.NoError(t, err, "Error funding Chainlink nodes") - - linkToken, err := contractDeployer.DeployLinkTokenContract() - require.NoError(t, err, "Error deploying LINK token") - - registry, registrar := actions.DeployAutoOCRRegistryAndRegistrar( - t, - registryVersion, - registryConfig, - defaultAmountOfUpkeeps, - linkToken, - contractDeployer, - chainClient, - ) + onlyStartRunner = testEnvironment.WillUseRemoteRunner() + if !onlyStartRunner { + chainClient, err = blockchain.NewEVMClient(network, testEnvironment) + require.NoError(t, err, "Error connecting to blockchain") + contractDeployer, err = contracts.NewContractDeployer(chainClient) + require.NoError(t, err, "Error building contract deployer") + chainlinkNodes, err = client.ConnectChainlinkNodes(testEnvironment) + require.NoError(t, err, "Error connecting to Chainlink nodes") + chainClient.ParallelTransactions(true) + + txCost, err := chainClient.EstimateCostForChainlinkOperations(1000) + require.NoError(t, err, "Error estimating cost for Chainlink Operations") + err = actions.FundChainlinkNodes(chainlinkNodes, chainClient, txCost) + require.NoError(t, err, "Error funding Chainlink nodes") + + linkToken, err = contractDeployer.DeployLinkTokenContract() + require.NoError(t, err, "Error deploying LINK token") + + registry, registrar = actions.DeployAutoOCRRegistryAndRegistrar( + t, + registryVersion, + registryConfig, + defaultAmountOfUpkeeps, + linkToken, + contractDeployer, + chainClient, + ) - actions.CreateOCRKeeperJobs(t, chainlinkNodes, registry.Address(), network.ChainID, 0) - nodesWithoutBootstrap := chainlinkNodes[1:] - ocrConfig := actions.BuildAutoOCR2ConfigVars(t, nodesWithoutBootstrap, registryConfig, registrar.Address(), 5*time.Second) - err = registry.SetConfig(automationDefaultRegistryConfig, ocrConfig) - require.NoError(t, err, "Registry config should be be set successfully") - require.NoError(t, chainClient.WaitForEvents(), "Waiting for config to be set") + actions.CreateOCRKeeperJobs(t, chainlinkNodes, registry.Address(), network.ChainID, 0) + nodesWithoutBootstrap := chainlinkNodes[1:] + ocrConfig := actions.BuildAutoOCR2ConfigVars(t, nodesWithoutBootstrap, registryConfig, registrar.Address(), 5*time.Second) + err = registry.SetConfig(automationDefaultRegistryConfig, ocrConfig) + require.NoError(t, err, "Registry config should be be set successfully") + require.NoError(t, chainClient.WaitForEvents(), "Waiting for config to be set") + // Register cleanup for any test + t.Cleanup(func() { + err := actions.TeardownSuite(t, testEnvironment, utils.ProjectRoot, chainlinkNodes, nil, chainClient) + require.NoError(t, err, "Error tearing down environment") + }) + } - return chainClient, chainlinkNodes, contractDeployer, linkToken, registry, registrar + return chainClient, chainlinkNodes, contractDeployer, linkToken, registry, registrar, onlyStartRunner } diff --git a/integration-tests/smoke/cron_test.go b/integration-tests/smoke/cron_test.go index 9018d024b86..3c7c8144407 100644 --- a/integration-tests/smoke/cron_test.go +++ b/integration-tests/smoke/cron_test.go @@ -24,6 +24,9 @@ import ( func TestCronBasic(t *testing.T) { t.Parallel() testEnvironment := setupCronTest(t) + if testEnvironment.WillUseRemoteRunner() { + return + } chainlinkNodes, err := client.ConnectChainlinkNodes(testEnvironment) require.NoError(t, err, "Connecting to chainlink nodes shouldn't fail") @@ -77,6 +80,7 @@ func setupCronTest(t *testing.T) (testEnvironment *environment.Environment) { } testEnvironment = environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-cron-%s", strings.ReplaceAll(strings.ToLower(network.Name), " ", "-")), + Test: t, }). AddHelm(mockservercfg.New(nil)). AddHelm(mockserver.New(nil)). diff --git a/integration-tests/smoke/flux_test.go b/integration-tests/smoke/flux_test.go index ce2611e31f2..d4b5727a101 100644 --- a/integration-tests/smoke/flux_test.go +++ b/integration-tests/smoke/flux_test.go @@ -31,6 +31,9 @@ import ( func TestFluxBasic(t *testing.T) { t.Parallel() testEnvironment, testNetwork := setupFluxTest(t) + if testEnvironment.WillUseRemoteRunner() { + return + } chainClient, err := blockchain.NewEVMClient(testNetwork, testEnvironment) require.NoError(t, err, "Connecting to blockchain nodes shouldn't fail") @@ -170,6 +173,7 @@ func setupFluxTest(t *testing.T) (testEnvironment *environment.Environment, test Enabled = true` testEnvironment = environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-flux-%s", strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-")), + Test: t, }). AddHelm(mockservercfg.New(nil)). AddHelm(mockserver.New(nil)). diff --git a/integration-tests/smoke/forwarder_ocr_test.go b/integration-tests/smoke/forwarder_ocr_test.go index e5d50918716..286457786b2 100644 --- a/integration-tests/smoke/forwarder_ocr_test.go +++ b/integration-tests/smoke/forwarder_ocr_test.go @@ -28,6 +28,9 @@ import ( func TestForwarderOCRBasic(t *testing.T) { t.Parallel() testEnvironment, testNetwork := setupForwarderOCRTest(t) + if testEnvironment.WillUseRemoteRunner() { + return + } chainClient, err := blockchain.NewEVMClient(testNetwork, testEnvironment) require.NoError(t, err, "Connecting to blockchain nodes shouldn't fail") @@ -118,6 +121,7 @@ ListenPort = 6690` ForwardersEnabled = true` testEnvironment = environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-ocr-forwarder-%s", strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-")), + Test: t, }). AddHelm(mockservercfg.New(nil)). AddHelm(mockserver.New(nil)). diff --git a/integration-tests/smoke/keeper_test.go b/integration-tests/smoke/keeper_test.go index a85a3256c02..2b43a87519b 100644 --- a/integration-tests/smoke/keeper_test.go +++ b/integration-tests/smoke/keeper_test.go @@ -6,7 +6,6 @@ import ( "math/big" "strconv" "strings" - "sync" "testing" "github.com/ethereum/go-ethereum/common" @@ -96,7 +95,10 @@ func TestKeeperBasicSmoke(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "basic-smoke") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "basic-smoke") + if onlyStartRunner { + return + } registry, _, consumers, upkeepIDs := actions.DeployKeeperContracts( t, registryVersion, @@ -170,7 +172,10 @@ func TestKeeperBlockCountPerTurn(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "bcpt") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "bcpt") + if onlyStartRunner { + return + } registry, _, consumers, upkeepIDs := actions.DeployKeeperContracts( t, registryVersion, @@ -270,7 +275,10 @@ func TestKeeperSimulation(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "simulation") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "simulation") + if onlyStartRunner { + return + } registry, _, consumersPerformance, upkeepIDs := actions.DeployPerformanceKeeperContracts( t, registryVersion, @@ -341,7 +349,10 @@ func TestKeeperCheckPerformGasLimit(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "gas-limit") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "gas-limit") + if onlyStartRunner { + return + } registry, _, consumersPerformance, upkeepIDs := actions.DeployPerformanceKeeperContracts( t, registryVersion, @@ -452,7 +463,10 @@ func TestKeeperRegisterUpkeep(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "register-upkeep") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "register-upkeep") + if onlyStartRunner { + return + } registry, registrar, consumers, upkeepIDs := actions.DeployKeeperContracts( t, registryVersion, @@ -537,7 +551,10 @@ func TestKeeperAddFunds(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "add-funds") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "add-funds") + if onlyStartRunner { + return + } registry, _, consumers, upkeepIDs := actions.DeployKeeperContracts( t, registryVersion, @@ -599,7 +616,10 @@ func TestKeeperRemove(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "remove") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "remove") + if onlyStartRunner { + return + } registry, _, consumers, upkeepIDs := actions.DeployKeeperContracts( t, registryVersion, @@ -674,7 +694,10 @@ func TestKeeperPauseRegistry(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "pause-registry") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "pause-registry") + if onlyStartRunner { + return + } registry, _, consumers, upkeepIDs := actions.DeployKeeperContracts( t, registryVersion, @@ -732,7 +755,10 @@ func TestKeeperPauseRegistry(t *testing.T) { func TestKeeperMigrateRegistry(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "migrate-registry") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "migrate-registry") + if onlyStartRunner { + return + } registry, _, consumers, upkeepIDs := actions.DeployKeeperContracts( t, ethereum.RegistryVersion_1_2, @@ -820,7 +846,10 @@ func TestKeeperNodeDown(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "node-down") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "node-down") + if onlyStartRunner { + return + } registry, _, consumers, upkeepIDs := actions.DeployKeeperContracts( t, registryVersion, @@ -913,7 +942,10 @@ func TestKeeperNodeDown(t *testing.T) { func TestKeeperPauseUnPauseUpkeep(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "pause-upkeep") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "pause-upkeep") + if onlyStartRunner { + return + } registry, _, consumers, upkeepIDs := actions.DeployKeeperContracts( t, ethereum.RegistryVersion_1_3, @@ -999,7 +1031,10 @@ func TestKeeperPauseUnPauseUpkeep(t *testing.T) { func TestKeeperUpdateCheckData(t *testing.T) { t.Parallel() - chainClient, chainlinkNodes, contractDeployer, linkToken := setupKeeperTest(t, "pause-upkeep") + chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner := setupKeeperTest(t, "pause-upkeep") + if onlyStartRunner { + return + } registry, _, performDataChecker, upkeepIDs := actions.DeployPerformDataCheckerContracts( t, ethereum.RegistryVersion_1_3, @@ -1056,16 +1091,15 @@ func TestKeeperUpdateCheckData(t *testing.T) { }, "3m", "1s").Should(gomega.Succeed()) } -var setupMu sync.Mutex - func setupKeeperTest( t *testing.T, testName string, ) ( - blockchain.EVMClient, - []*client.Chainlink, - contracts.ContractDeployer, - contracts.LinkToken, + chainClient blockchain.EVMClient, + chainlinkNodes []*client.Chainlink, + contractDeployer contracts.ContractDeployer, + linkToken contracts.LinkToken, + onlyStartRunner bool, ) { network := networks.SelectedNetwork evmConfig := eth.New(nil) @@ -1078,8 +1112,10 @@ func setupKeeperTest( } networkName := strings.ReplaceAll(strings.ToLower(network.Name), " ", "-") testEnvironment := environment.New( - &environment.Config{NamespacePrefix: fmt.Sprintf("smoke-keeper-%s-%s", testName, networkName)}, - ). + &environment.Config{ + NamespacePrefix: fmt.Sprintf("smoke-keeper-%s-%s", testName, networkName), + Test: t, + }). AddHelm(mockservercfg.New(nil)). AddHelm(mockserver.New(nil)). AddHelm(evmConfig). @@ -1089,29 +1125,31 @@ func setupKeeperTest( })) err := testEnvironment.Run() require.NoError(t, err, "Error deploying test environment") + onlyStartRunner = testEnvironment.WillUseRemoteRunner() + if !onlyStartRunner { + chainClient, err = blockchain.NewEVMClient(network, testEnvironment) + require.NoError(t, err, "Connecting to blockchain nodes shouldn't fail") + contractDeployer, err = contracts.NewContractDeployer(chainClient) + require.NoError(t, err, "Deploying contracts shouldn't fail") + chainlinkNodes, err = client.ConnectChainlinkNodes(testEnvironment) + require.NoError(t, err, "Connecting to chainlink nodes shouldn't fail") + chainClient.ParallelTransactions(true) + + // Register cleanup for any test + t.Cleanup(func() { + err := actions.TeardownSuite(t, testEnvironment, utils.ProjectRoot, chainlinkNodes, nil, chainClient) + require.NoError(t, err, "Error tearing down environment") + }) - chainClient, err := blockchain.NewEVMClient(network, testEnvironment) - require.NoError(t, err, "Connecting to blockchain nodes shouldn't fail") - contractDeployer, err := contracts.NewContractDeployer(chainClient) - require.NoError(t, err, "Deploying contracts shouldn't fail") - chainlinkNodes, err := client.ConnectChainlinkNodes(testEnvironment) - require.NoError(t, err, "Connecting to chainlink nodes shouldn't fail") - chainClient.ParallelTransactions(true) - - // Register cleanup for any test - t.Cleanup(func() { - err := actions.TeardownSuite(t, testEnvironment, utils.ProjectRoot, chainlinkNodes, nil, chainClient) - require.NoError(t, err, "Error tearing down environment") - }) - - err = actions.FundChainlinkNodes(chainlinkNodes, chainClient, big.NewFloat(.5)) - require.NoError(t, err, "Funding Chainlink nodes shouldn't fail") + err = actions.FundChainlinkNodes(chainlinkNodes, chainClient, big.NewFloat(.5)) + require.NoError(t, err, "Funding Chainlink nodes shouldn't fail") - linkToken, err := contractDeployer.DeployLinkTokenContract() - require.NoError(t, err, "Deploying Link Token Contract shouldn't fail") + linkToken, err = contractDeployer.DeployLinkTokenContract() + require.NoError(t, err, "Deploying Link Token Contract shouldn't fail") - err = chainClient.WaitForEvents() - require.NoError(t, err, "Error waiting for events") + err = chainClient.WaitForEvents() + require.NoError(t, err, "Error waiting for events") + } - return chainClient, chainlinkNodes, contractDeployer, linkToken + return chainClient, chainlinkNodes, contractDeployer, linkToken, onlyStartRunner } diff --git a/integration-tests/smoke/ocr2vrf_test.go b/integration-tests/smoke/ocr2vrf_test.go index 370a78d12e3..e7fe978a56a 100644 --- a/integration-tests/smoke/ocr2vrf_test.go +++ b/integration-tests/smoke/ocr2vrf_test.go @@ -37,6 +37,9 @@ func TestOCR2VRFBasic(t *testing.T) { t.Parallel() testEnvironment, testNetwork := setupOCR2VRFTest(t) + if testEnvironment.WillUseRemoteRunner() { + return + } chainClient, err := blockchain.NewEVMClient(testNetwork, testEnvironment) require.NoError(t, err) @@ -333,6 +336,7 @@ FeeCapDefault = 100000000000` testEnvironment = environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-ocr2vrf-%s", strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-")), + Test: t, }). AddHelm(evmConfig). AddHelm(chainlink.New(0, map[string]interface{}{ diff --git a/integration-tests/smoke/ocr_test.go b/integration-tests/smoke/ocr_test.go index 6fc2fe25484..dd535da6af2 100644 --- a/integration-tests/smoke/ocr_test.go +++ b/integration-tests/smoke/ocr_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/rs/zerolog/log" "github.com/smartcontractkit/chainlink-env/environment" "github.com/smartcontractkit/chainlink-env/pkg/helm/chainlink" "github.com/smartcontractkit/chainlink-env/pkg/helm/ethereum" @@ -27,6 +28,9 @@ import ( func TestOCRBasic(t *testing.T) { t.Parallel() testEnvironment, testNetwork := setupOCRTest(t) + if testEnvironment.WillUseRemoteRunner() { + return + } chainClient, err := blockchain.NewEVMClient(testNetwork, testEnvironment) require.NoError(t, err, "Connecting to blockchain nodes shouldn't fail") @@ -68,6 +72,7 @@ func TestOCRBasic(t *testing.T) { answer, err = ocrInstances[0].GetLatestAnswer(context.Background()) require.NoError(t, err, "Error getting latest OCR answer") require.Equal(t, int64(10), answer.Int64(), "Expected latest answer from OCR contract to be 10 but got %d", answer.Int64()) + log.Debug().Str("", "").Msg("End of the test") } func setupOCRTest(t *testing.T) (testEnvironment *environment.Environment, testNetwork blockchain.EVMNetwork) { @@ -82,6 +87,7 @@ func setupOCRTest(t *testing.T) (testEnvironment *environment.Environment, testN } testEnvironment = environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-ocr-%s", strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-")), + Test: t, }). AddHelm(mockservercfg.New(nil)). AddHelm(mockserver.New(nil)). diff --git a/integration-tests/smoke/runlog_test.go b/integration-tests/smoke/runlog_test.go index 3636454d309..64e6e73df48 100644 --- a/integration-tests/smoke/runlog_test.go +++ b/integration-tests/smoke/runlog_test.go @@ -29,6 +29,9 @@ import ( func TestRunLogBasic(t *testing.T) { t.Parallel() testEnvironment, testNetwork := setupRunLogTest(t) + if testEnvironment.WillUseRemoteRunner() { + return + } chainClient, err := blockchain.NewEVMClient(testNetwork, testEnvironment) require.NoError(t, err, "Connecting to blockchain nodes shouldn't fail") @@ -120,6 +123,7 @@ func setupRunLogTest(t *testing.T) (testEnvironment *environment.Environment, te } testEnvironment = environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-runlog-%s", strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-")), + Test: t, }). AddHelm(mockservercfg.New(nil)). AddHelm(mockserver.New(nil)). diff --git a/integration-tests/smoke/vrf_test.go b/integration-tests/smoke/vrf_test.go index 8836ac00591..e7a7f32f709 100644 --- a/integration-tests/smoke/vrf_test.go +++ b/integration-tests/smoke/vrf_test.go @@ -28,6 +28,9 @@ import ( func TestVRFBasic(t *testing.T) { t.Parallel() testEnvironment, testNetwork := setupVRFTest(t) + if testEnvironment.WillUseRemoteRunner() { + return + } chainClient, err := blockchain.NewEVMClient(testNetwork, testEnvironment) require.NoError(t, err, "Connecting client shouldn't fail") @@ -136,6 +139,7 @@ func setupVRFTest(t *testing.T) (testEnvironment *environment.Environment, testN } testEnvironment = environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-vrf-%s", strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-")), + Test: t, }). AddHelm(evmConfig). AddHelm(chainlink.New(0, map[string]interface{}{ diff --git a/integration-tests/smoke/vrfv2_test.go b/integration-tests/smoke/vrfv2_test.go index e84917eaaa4..bb35c6adcb1 100644 --- a/integration-tests/smoke/vrfv2_test.go +++ b/integration-tests/smoke/vrfv2_test.go @@ -34,6 +34,9 @@ func TestVRFv2Basic(t *testing.T) { t.Parallel() testEnvironment, testNetwork := setupVRFv2Test(t) + if testEnvironment.WillUseRemoteRunner() { + return + } chainClient, err := blockchain.NewEVMClient(testNetwork, testEnvironment) require.NoError(t, err) @@ -170,6 +173,7 @@ PriceMax = 100000000000 FeeCapDefault = 100000000000` testEnvironment = environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-vrfv2-%s", strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-")), + Test: t, }). AddHelm(evmConfig). AddHelm(chainlink.New(0, map[string]interface{}{ diff --git a/integration-tests/soak/soak_runner_test.go b/integration-tests/soak/soak_runner_test.go index 13665124f47..ff455ce5c5e 100644 --- a/integration-tests/soak/soak_runner_test.go +++ b/integration-tests/soak/soak_runner_test.go @@ -9,6 +9,7 @@ import ( "github.com/rs/zerolog/log" "github.com/stretchr/testify/require" + "github.com/smartcontractkit/chainlink-env/config" "github.com/smartcontractkit/chainlink-env/environment" "github.com/smartcontractkit/chainlink-env/pkg/helm/chainlink" "github.com/smartcontractkit/chainlink-env/pkg/helm/ethereum" @@ -120,6 +121,7 @@ func soakTestHelper( for key, value := range activeEVMNetwork.ToMap() { remoteRunnerValues[key] = value } + remoteRunnerValues[config.EnvVarInsideK8s] = "true" remoteRunnerWrapper := map[string]interface{}{"remote_test_runner": remoteRunnerValues} err := testEnvironment. diff --git a/integration-tests/test.Dockerfile b/integration-tests/test.Dockerfile new file mode 100644 index 00000000000..fc6eefd650c --- /dev/null +++ b/integration-tests/test.Dockerfile @@ -0,0 +1,10 @@ +ARG BASE_IMAGE +ARG IMAGE_VERSION=latest +FROM ${BASE_IMAGE}:${IMAGE_VERSION} + +ARG SUITES=chaos migration performance reorg smoke soak benchmark + +COPY . testdir/ +WORKDIR /go/testdir +RUN /go/testdir/integration-tests/scripts/buildTests "${SUITES}" +ENTRYPOINT ["/go/testdir/integration-tests/scripts/entrypoint"] diff --git a/integration-tests/test.Dockerfile.dockerignore b/integration-tests/test.Dockerfile.dockerignore new file mode 100644 index 00000000000..cbd0d73604d --- /dev/null +++ b/integration-tests/test.Dockerfile.dockerignore @@ -0,0 +1,15 @@ +.DS_Store +.github +.envrc +.vscode/ +*.log +node_modules/ +**/node_modules/ +vendor/ +tmp/ + +contracts/node_modules +examples/ +tools/ +docs/ +operator_ui/