Skip to content

Commit

Permalink
QA-134 Add Isomorphic Deployment ability to core smoke tests (#7951)
Browse files Browse the repository at this point in the history
* testing out new test deployment build

* bump to latest chainlink-env isomorphic branch

* switch to compilation in docker

* network connection debugging

* more debugging

* additional ginkgo args

* add sleep retry loop to test if first connection is just problematic

* wrap test in if for local vs remote

* merge conflict fixes and docker image updates

* Merge conflict fixes

* Fix integration-tests chainlink image version

* add test image publish on merge to develop

* Use existing version

* Remove the need to add a test cleanup specifically for remote run tests

* bump chainlink-env

* Add fixes for soak testing

* another clenv bump

* fix for benchmark test

* Update ocr2vrf types

* Use chainlink-env v0.3.0
  • Loading branch information
tateexon authored Jan 19, 2023
1 parent 2aeba0a commit dcca118
Show file tree
Hide file tree
Showing 30 changed files with 456 additions and 144 deletions.
43 changes: 43 additions & 0 deletions .github/workflows/integration-tests-publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: Integration Tests Publish
# Publish the compiled integration tests

on:
push:
branches:
- develop

env:
ECR_TAG: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:develop

jobs:
publish-integration-test-image:
environment: integration
permissions:
id-token: write
contents: read
name: Publish Integration Test Image
runs-on: ubuntu-latest
steps:
- name: Collect Metrics
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@v1
with:
basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: Publish Integration Test Image
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: build test runner
uses: smartcontractkit/chainlink-github-actions/docker/build-push@e72f0a768ac934afce498a802de893d89b12802f # v2.1.1
with:
tags: ${{ env.ECR_TAG }}
file: ./integration-tests/test.Dockerfile
build-args: |
BASE_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/test-base-image
IMAGE_VERSION=v0.3.0
SUITES="chaos migration performance reorg smoke soak benchmark"
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
46 changes: 45 additions & 1 deletion .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ concurrency:
group: integration-tests-chainlink-${{ github.ref }}
cancel-in-progress: true

env:
CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:ci.${{ github.sha }}

jobs:
changes:
environment: integration
Expand Down Expand Up @@ -35,6 +39,7 @@ jobs:
continue-on-error: true
outputs:
src: ${{ steps.changes.outputs.src }}

build-chainlink:
environment: integration
permissions:
Expand Down Expand Up @@ -67,6 +72,40 @@ jobs:
this-job-name: Build Chainlink Image
continue-on-error: true

build-test-image:
environment: integration
permissions:
id-token: write
contents: read
name: Build Test Image
runs-on: ubuntu20.04-16cores-64GB
needs: [changes]
steps:
- name: Collect Metrics
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@v1
with:
basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: Build Test Image
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: build test runner
uses: smartcontractkit/chainlink-github-actions/docker/build-push@e72f0a768ac934afce498a802de893d89b12802f # v2.1.1
with:
tags: ${{ env.ENV_JOB_IMAGE }}
file: ./integration-tests/test.Dockerfile
build-args: |
BASE_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/test-base-image
IMAGE_VERSION=v0.3.0
SUITES="smoke"
#SUITES="chaos migration performance reorg smoke soak"
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

eth-smoke-tests-matrix:
environment: integration
permissions:
Expand All @@ -75,7 +114,7 @@ jobs:
id-token: write
contents: read
name: ETH Smoke Tests Matrix
needs: [build-chainlink, changes]
needs: [build-chainlink, changes, build-test-image]
env:
SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
CHAINLINK_COMMIT_SHA: ${{ github.sha }}
Expand Down Expand Up @@ -126,6 +165,11 @@ jobs:
- name: Run Tests
if: needs.changes.outputs.src == 'true'
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e72f0a768ac934afce498a802de893d89b12802f # v2.1.1
env:
TEST_SUITE: smoke
TEST_ARGS: -test.timeout 30m
CHAINLINK_IMAGE: ${{ env.CL_ECR }}
CHAINLINK_VERSION: latest.${{ github.sha }}
with:
test_command_to_run: make test_need_operator_assets && cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ./smoke/${{ matrix.product.name }}_test.go 2>&1 | tee /tmp/gotest.log | gotestfmt
test_download_vendor_packages_command: make gomod
Expand Down
3 changes: 1 addition & 2 deletions .tool-versions
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ golang 1.19.4
mockery 2.14.0
nodejs 16.13.2
postgres 13.3
ginkgo 2.5.1
helm 3.9.4
helm 3.10.3
zig 0.10.0
golangci-lint 1.50.1
1 change: 1 addition & 0 deletions CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
# CI/CD
/.github/** @alexroan @chainchad @javuto @jkongie @jmank88 @samsondav
/.github/workflows/integration-tests.yml @kalverra @tateexon @skudasov @anieeg
/.github/workflows/integration-tests-publish.yml @kalverra @tateexon @skudasov @anieeg
/.github/workflows/integration-chaos-tests.yml @kalverra @tateexon @skudasov @anieeg
/.github/workflows/performance-tests.yml @kalverra @tateexon @skudasov @anieeg
/core/chainlink.Dockerfile @smartcontractkit/prodsec-public
Expand Down
7 changes: 4 additions & 3 deletions integration-tests/.tool-versions
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
golang 1.19.4
k3d 5.4.4
kubectl 1.25.3
golang 1.19.5
k3d 5.4.6
kubectl 1.25.5
nodejs 18.13.0
9 changes: 9 additions & 0 deletions integration-tests/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,15 @@ install_gotestfmt:
go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
set -euo pipefail

# Builds the test image
# tag: the tag for the test image being built, example: tag=tate
# base_tag: the tag for the base-test-image to use, example: base_tag=latest
# suite: the test suites to build into the image, example: suite="chaos soak smoke reorg migration performance"
# push: set to true if you want the image pushed or leave blank if not, example: push=true
.PHONY: build_test_image
build_test_image:
./scripts/buildTestImage $(tag) $(base_tag) "$(suite)" $(push)

## All commands will use 16 threads to run tests in parallel. To change this, use -test.parallel n

# Smoke
Expand Down
43 changes: 29 additions & 14 deletions integration-tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,6 @@ Prerequisites to run the tests.
[Install](https://go.dev/doc/install)
</details>

<details>
<summary>Install Ginkgo</summary>

[Ginkgo](https://onsi.github.io/ginkgo/) is the testing framework we use to compile and run our tests. It comes with a lot of handy testing setups and goodies on top of the standard Go testing packages.

`go install github.com/onsi/ginkgo/v2/ginkgo`
</details>

<details>
<summary>Install NodeJS</summary>

Expand Down Expand Up @@ -76,17 +68,40 @@ make test_smoke_simulated # Run all smoke tests on a simulated network
Run all smoke tests in parallel, only using simulated blockchains. *Note: As of now, you can only run tests in parallel on simulated chains, not on live ones. Running on parallel tests on live chains will give errors*

```sh
make test_smoke_simulated args="-nodes=<number-of-parallel-tests>"
make test_smoke_simulated args="-test.parallel=<number-of-parallel-tests>"
```

You can also run specific tests using `make test_smoke` and a `focus` tag.
You can also run specific tests and debug tests in vscode by setting up your .vscode/settings.json with this information. Just replace all the "<put your ...>" with your information before running a test.

```json
{
"makefile.extensionOutputFolder": "./.vscode",
"go.testEnvVars": {
"LOG_LEVEL": "debug",
"SELECTED_NETWORKS": "SIMULATED,SIMULATED_1,SIMULATED_2",
"CHAINLINK_IMAGE":"<put your account number here>.dkr.ecr.us-west-2.amazonaws.com/chainlink",
"CHAINLINK_VERSION":"develop",
"CHAINLINK_ENV_USER":"<put your name>",
"TEST_LOG_LEVEL":"debug",
"AWS_ACCESS_KEY_ID":"<put your access key id here>",
"AWS_SECRET_ACCESS_KEY":"<put your access key here>",
"AWS_SESSION_TOKEN":"<put your token here>"
},
"go.testTimeout": "900s"
}
```

You can also run your tests inside of kubernetes instead of from locally to reduce local resource usage and the number of ports that get forwarded to the cluster. This is not recommended for normal developement since building and pushing the image can be time heavy depending on your internet upload speeds. To do this you will want to either pull down an already built chainlink-tests image or build one yourself. To build and push one yourself you can run:
```sh
make test_smoke args="-focus=@ocr" # Runs all the ocr smoke tests
make test_smoke args="-focus=@keeper" # Runs all smoke tests for keepers
make build_test_image tag=<a tag for your image> base_tag=latest suite="smoke soak chaos reorg migration performance" push=true
```

[Check out](https://onsi.github.io/ginkgo/#description-based-filtering) how Ginkgo handles focus and skip tags if you're looking for more precise behavior.
Once that is done building you can add this to your go.testEnvVars in .vscode/settings.json with the correct account number and tag filled out.
```json
"TEST_SUITE": "smoke",
"TEST_ARGS": "-test.timeout 30m",
"ENV_JOB_IMAGE":"<account number>.dkr.ecr.us-west-2.amazonaws.com/chainlink-env-tests:<tag you used in the build step>",
```
Once that is done you can run/debug your test using the vscode test view just like normal.

### Soak

Expand Down
25 changes: 4 additions & 21 deletions integration-tests/actions/actions.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"math/big"
"os"
"strings"
"testing"

Expand Down Expand Up @@ -230,21 +229,18 @@ func TeardownSuite(
optionalTestReporter testreporters.TestReporter, // Optionally pass in a test reporter to log further metrics
clients ...blockchain.EVMClient,
) error {
keepEnvs := os.Getenv("KEEP_ENVIRONMENTS")
if keepEnvs == "" {
keepEnvs = "NEVER"
}

if err := testreporters.WriteTeardownLogs(t, env, optionalTestReporter); err != nil {
return errors.Wrap(err, "Error dumping environment logs, leaving environment running for manual retrieval")
}
for _, c := range clients {
if c != nil && chainlinkNodes != nil && len(chainlinkNodes) > 0 {
if err := returnFunds(chainlinkNodes, c); err != nil {
// This printed line is required for tests that use real funds to propagate the failure
// out to the system running the test. Do not remove
fmt.Println(environment.FAILED_FUND_RETURN)
log.Error().Err(err).Str("Namespace", env.Cfg.Namespace).
Msg("Error attempting to return funds from chainlink nodes to network's default wallet. " +
"Environment is left running so you can try manually!")
keepEnvs = "ALWAYS"
}
} else {
log.Info().Msg("Successfully returned funds from chainlink nodes to default network wallets")
Expand All @@ -258,20 +254,7 @@ func TeardownSuite(
}
}

switch strings.ToUpper(keepEnvs) {
case "ALWAYS":
return nil
case "ONFAIL":
if !t.Failed() {
return env.Shutdown()
}
case "NEVER":
return env.Shutdown()
default:
log.Warn().Str("Invalid Keep Value", keepEnvs).
Msg("Invalid 'keep_environments' value, see the KEEP_ENVIRONMENTS env var")
}
return nil
return env.Shutdown()
}

// TeardownRemoteSuite is used when running a test within a remote-test-runner, like for long-running performance and
Expand Down
2 changes: 2 additions & 0 deletions integration-tests/benchmark/benchmark_runner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"time"

"github.com/rs/zerolog/log"
"github.com/smartcontractkit/chainlink-env/config"
"github.com/smartcontractkit/chainlink-env/pkg/cdk8s/blockscout"

"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -183,6 +184,7 @@ func benchmarkTestHelper(
for key, value := range activeEVMNetwork.ToMap() {
remoteRunnerValues[key] = value
}
remoteRunnerValues[config.EnvVarInsideK8s] = "true"
remoteRunnerWrapper := map[string]interface{}{
"remote_test_runner": remoteRunnerValues,
}
Expand Down
1 change: 1 addition & 0 deletions integration-tests/client/chainlink.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ func NewChainlink(c *ChainlinkConfig) (*Chainlink, error) {
session := &Session{Email: c.Email, Password: c.Password}
resp, err := rc.R().SetBody(session).Post("/sessions")
if err != nil {
log.Info().Interface("session", session).Msg("session used")
return nil, err
}
rc.SetCookies(resp.Cookies())
Expand Down
4 changes: 2 additions & 2 deletions integration-tests/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ require (
github.com/satori/go.uuid v1.2.0
github.com/slack-go/slack v0.11.4
github.com/smartcontractkit/chainlink v1.10.0
github.com/smartcontractkit/chainlink-env v0.2.60
github.com/smartcontractkit/chainlink-env v0.3.0
github.com/smartcontractkit/chainlink-testing-framework v1.9.0
github.com/smartcontractkit/libocr v0.0.0-20221209172631-568a30f68407
github.com/smartcontractkit/ocr2keepers v0.6.6
github.com/smartcontractkit/ocr2vrf v0.0.0-20230117141905-c3a5c0467fd4
github.com/smartcontractkit/ocr2vrf v0.0.0-20230117145849-09a91a822ee2
github.com/stretchr/testify v1.8.1
github.com/umbracle/ethgo v0.1.3
go.dedis.ch/kyber/v3 v3.0.14
Expand Down
8 changes: 4 additions & 4 deletions integration-tests/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1599,8 +1599,8 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/slack-go/slack v0.11.4 h1:ojSa7KlPm3PqY2AomX4VTxEsK5eci5JaxCjlzGV5zoM=
github.com/slack-go/slack v0.11.4/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
github.com/smartcontractkit/chainlink-env v0.2.60 h1:Vu7GBh9Z25pWR9kyZacBUfu7MHUyYvUsK5ub77MVuco=
github.com/smartcontractkit/chainlink-env v0.2.60/go.mod h1:zspYFdXS57CzKkpcLkFEPuMQHOmI3L9H5idoPxobxRM=
github.com/smartcontractkit/chainlink-env v0.3.0 h1:HfyjiXeWUEDTI5vM7asSpTzqrx/9R71915MNEta+Ihw=
github.com/smartcontractkit/chainlink-env v0.3.0/go.mod h1:zspYFdXS57CzKkpcLkFEPuMQHOmI3L9H5idoPxobxRM=
github.com/smartcontractkit/chainlink-relay v0.1.6-0.20221025223751-9b407cff57eb h1:NF6//JILgK8AeLkknJFEVsVRt+VqwNnxJ4SLpHKje9c=
github.com/smartcontractkit/chainlink-relay v0.1.6-0.20221025223751-9b407cff57eb/go.mod h1:v/QSrVm3z4/aPz/PLB6da05B/r4MHZy0/jder7iPxkQ=
github.com/smartcontractkit/chainlink-solana v1.0.2-0.20220930034647-edd5a863b876 h1:uctLwzPqXUbWWcOiZaltKNtb2XfIDVE1yQ04uLZ3N7Q=
Expand All @@ -1613,8 +1613,8 @@ github.com/smartcontractkit/libocr v0.0.0-20221209172631-568a30f68407 h1:P3dhh6U
github.com/smartcontractkit/libocr v0.0.0-20221209172631-568a30f68407/go.mod h1:5JnCHuYgmIP9ZyXzgAfI5Iwu0WxBtBKp+ApeT5o1Cjw=
github.com/smartcontractkit/ocr2keepers v0.6.6 h1:21wVK5HGHLcfHo4HU4QW8wLCvhENFMhkXdqxMcHPgkU=
github.com/smartcontractkit/ocr2keepers v0.6.6/go.mod h1:0y0wgX0OQTyOb04f823ifLvzsca+rsyE3ngGnmG8zxU=
github.com/smartcontractkit/ocr2vrf v0.0.0-20230117141905-c3a5c0467fd4 h1:ZS37mHqFOpbVj45sJ3bJQ2UJzRxhBkxFflyPNCT5z4A=
github.com/smartcontractkit/ocr2vrf v0.0.0-20230117141905-c3a5c0467fd4/go.mod h1:NKkp8yf3trq+hJe/gn2Q1dd4abZvcYavUKPzYJCgVew=
github.com/smartcontractkit/ocr2vrf v0.0.0-20230117145849-09a91a822ee2 h1:xiaI621shLKa4oK81bMMZdQZq5s08iWSNzWRM7LaqoA=
github.com/smartcontractkit/ocr2vrf v0.0.0-20230117145849-09a91a822ee2/go.mod h1:NKkp8yf3trq+hJe/gn2Q1dd4abZvcYavUKPzYJCgVew=
github.com/smartcontractkit/sqlx v1.3.5-0.20210805004948-4be295aacbeb h1:OMaBUb4X9IFPLbGbCHsMU+kw/BPCrewaVwWGIBc0I4A=
github.com/smartcontractkit/sqlx v1.3.5-0.20210805004948-4be295aacbeb/go.mod h1:HNUu4cJekUdsJbwRBCiOybtkPJEfGRELQPe2tkoDEyk=
github.com/smartcontractkit/terra.go v1.0.3-0.20220108002221-62b39252ee16 h1:k+E0RKzVSG1QpxXakNUtcGUhq4ZMe0MAJ5Awg/l9oSc=
Expand Down
3 changes: 2 additions & 1 deletion integration-tests/known_networks.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,8 @@ var (
func determineSelectedNetworks() []blockchain.EVMNetwork {
logging.Init()
selectedNetworks := make([]blockchain.EVMNetwork, 0)
setNetworkNames := strings.Split(strings.ToUpper(os.Getenv("SELECTED_NETWORKS")), ",")
rawSelectedNetworks := strings.ToUpper(os.Getenv("SELECTED_NETWORKS"))
setNetworkNames := strings.Split(rawSelectedNetworks, ",")

for _, setNetworkName := range setNetworkNames {
if chosenNetwork, valid := mappedNetworks[setNetworkName]; valid {
Expand Down
41 changes: 41 additions & 0 deletions integration-tests/scripts/buildTestImage
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
#!/usr/bin/env bash

# build test binaries
# accepts a single space separated argument of the folders to build

set -ex

# get this scripts directory
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)

cd "$SCRIPT_DIR"/../../ || exit 1

TAG_VERSION="${1}"
BASE_IMAGE_VERSION="${2}"
SUITES=$3
DEFAULT_SUITES="smoke soak chaos"
ACCOUNT=$(aws sts get-caller-identity | jq -r .Account)
AWS_BASE="${ACCOUNT}".dkr.ecr.us-west-2.amazonaws.com
TAG="${AWS_BASE}"/chainlink-tests:"${TAG_VERSION}"
BASE_IMAGE="${AWS_BASE}"/test-base-image

if [ "${TAG_VERSION}" = "" ]; then
echo "Need an argument for the image tag version in argument 1"
exit 1
fi

if [ "${BASE_IMAGE_VERSION}" = "" ]; then
echo "Need an argument for the test-base-image tag version in argument 2"
exit 1
fi

if [ "${SUITES}" = "" ]; then
echo "SUITES not set, using defaults \"${DEFAULT_SUITES}\""
SUITES=${DEFAULT_SUITES}
fi

aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin
docker build -t "${TAG}" -f "./integration-tests/test.Dockerfile" --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg IMAGE_VERSION="${BASE_IMAGE_VERSION}" --build-arg SUITES="${SUITES}" .
if "${4}" = "true"]; then
docker push "${TAG}"
fi
Loading

0 comments on commit dcca118

Please sign in to comment.