diff --git a/integration-tests/actions/ocr2_helpers.go b/integration-tests/actions/ocr2_helpers.go index 1d386b158a0..842395bb25b 100644 --- a/integration-tests/actions/ocr2_helpers.go +++ b/integration-tests/actions/ocr2_helpers.go @@ -4,12 +4,15 @@ import ( "crypto/ed25519" "encoding/hex" "fmt" + "net/http" "strings" "time" + "github.com/avast/retry-go" "github.com/ethereum/go-ethereum/common" "github.com/google/uuid" "github.com/lib/pq" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" "gopkg.in/guregu/null.v4" @@ -192,6 +195,7 @@ func CreateOCRv2Jobs( mockServerValue int, // Value to get from the mock server when querying the path chainId int64, // EVM chain ID forwardingAllowed bool, + l zerolog.Logger, ) error { // Collect P2P ID bootstrapP2PIds, err := bootstrapNode.MustReadP2PKeys() @@ -218,6 +222,9 @@ func CreateOCRv2Jobs( } } + // Initialize map to store job IDs for each chainlink node + jobIDs := make(map[*nodeclient.ChainlinkK8sClient][]string) + for _, ocrInstance := range ocrInstances { bootstrapSpec := &nodeclient.OCR2TaskJobSpec{ Name: fmt.Sprintf("ocr2-bootstrap-%s", ocrInstance.Address()), @@ -284,10 +291,46 @@ func CreateOCRv2Jobs( P2PV2Bootstrappers: pq.StringArray{p2pV2Bootstrapper}, // bootstrap node key and address @bootstrap:6690 }, } - _, err = chainlinkNode.MustCreateJob(ocrSpec) + var ocrJob *nodeclient.Job + ocrJob, err = chainlinkNode.MustCreateJob(ocrSpec) if err != nil { return fmt.Errorf("creating OCR task job on OCR node have failed: %w", err) } + jobIDs[chainlinkNode] = append(jobIDs[chainlinkNode], ocrJob.Data.ID) // Store each job ID per node + } + } + l.Info().Msg("Verify OCRv2 jobs have been created") + for chainlinkNode, ids := range jobIDs { + for _, jobID := range ids { + err := retry.Do( + func() error { + _, resp, err := chainlinkNode.ReadJob(jobID) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected response status: %d", resp.StatusCode) + } + l.Info(). + Str("Node", chainlinkNode.PodName). + Str("Job ID", jobID). + Msg("OCRv2 job successfully created") + return nil + }, + retry.Attempts(4), + retry.Delay(time.Second*2), + retry.OnRetry(func(n uint, err error) { + l.Debug(). + Str("Node", chainlinkNode.PodName). + Str("Job ID", jobID). + Uint("Attempt", n+1). + Err(err). + Msg("Retrying job verification") + }), + ) + if err != nil { + l.Error().Err(err).Str("Node", chainlinkNode.PodName).Str("JobID", jobID).Msg("Failed to verify OCRv2 job creation") + } } } return nil diff --git a/integration-tests/go.mod b/integration-tests/go.mod index c9eced2ddfc..621c1c0941b 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -11,6 +11,7 @@ require ( dario.cat/mergo v1.0.1 github.com/AlekSi/pointer v1.1.0 github.com/Masterminds/semver/v3 v3.3.0 + github.com/avast/retry-go v3.0.0+incompatible github.com/avast/retry-go/v4 v4.6.0 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a @@ -102,7 +103,6 @@ require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/atombender/go-jsonschema v0.16.1-0.20240916205339-a74cd4e2851c // indirect - github.com/avast/retry-go v3.0.0+incompatible // indirect github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect github.com/aws/aws-sdk-go v1.54.19 // indirect github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index 20e1b4bf158..7a90c38fdd0 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -64,13 +64,14 @@ type OCRSoakTest struct { Config *tc.TestConfig TestReporter testreporters.OCRSoakTestReporter OperatorForwarderFlow bool - seth *seth.Client + sethClient *seth.Client OCRVersion string t *testing.T startTime time.Time timeLeft time.Duration startingBlockNum uint64 + startingValue int testEnvironment *environment.Environment namespace string log zerolog.Logger @@ -88,6 +89,8 @@ type OCRSoakTest struct { ocrV2Instances []contracts.OffchainAggregatorV2 ocrV2InstanceMap map[string]contracts.OffchainAggregatorV2 // address : instance + linkContract *contracts.EthereumLinkToken + rpcNetwork blockchain.EVMNetwork // network configuration for the blockchain node reorgHappened bool // flag to indicate if a reorg happened during the test gasSpikeSimulationHappened bool // flag to indicate if a gas spike simulation happened during the test @@ -280,107 +283,151 @@ func (o *OCRSoakTest) Environment() *environment.Environment { return o.testEnvironment } +// Setup initializes the OCR Soak Test by setting up clients, funding nodes, and deploying OCR contracts. func (o *OCRSoakTest) Setup(ocrTestConfig tt.OcrTestConfig) { + o.initializeClients() + o.deployLinkTokenContract(ocrTestConfig) + o.fundChainlinkNodes() + + o.startingValue = 5 + + var forwarders []common.Address + if o.OperatorForwarderFlow { + _, forwarders = o.deployForwarderContracts() + } + + o.setupOCRContracts(ocrTestConfig, forwarders) + o.log.Info().Msg("OCR Soak Test Setup Complete") +} + +// initializeClients sets up the Seth client, Chainlink nodes, and mock server. +func (o *OCRSoakTest) initializeClients() { sethClient, err := seth_utils.GetChainClient(o.Config, o.rpcNetwork) require.NoError(o.t, err, "Error creating seth client") - o.seth = sethClient + o.sethClient = sethClient nodes, err := nodeclient.ConnectChainlinkNodes(o.testEnvironment) require.NoError(o.t, err, "Connecting to chainlink nodes shouldn't fail") o.bootstrapNode, o.workerNodes = nodes[0], nodes[1:] + o.mockServer = ctf_client.ConnectMockServer(o.testEnvironment) require.NoError(o.t, err, "Creating mockserver clients shouldn't fail") +} - linkContract, err := actions.LinkTokenContract(o.log, sethClient, ocrTestConfig.GetActiveOCRConfig()) +func (o *OCRSoakTest) deployLinkTokenContract(ocrTestConfig tt.OcrTestConfig) { + linkContract, err := actions.LinkTokenContract(o.log, o.sethClient, ocrTestConfig.GetActiveOCRConfig()) require.NoError(o.t, err, "Error loading/deploying link token contract") + o.linkContract = linkContract +} - // Fund Chainlink nodes, excluding the bootstrap node +// fundChainlinkNodes funds the Chainlink worker nodes. +func (o *OCRSoakTest) fundChainlinkNodes() { o.log.Info().Float64("ETH amount per node", *o.Config.Common.ChainlinkNodeFunding).Msg("Funding Chainlink nodes") - err = actions.FundChainlinkNodesFromRootAddress(o.log, sethClient, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(o.workerNodes), big.NewFloat(*o.Config.Common.ChainlinkNodeFunding)) + err := actions.FundChainlinkNodesFromRootAddress(o.log, o.sethClient, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(o.workerNodes), big.NewFloat(*o.Config.Common.ChainlinkNodeFunding)) require.NoError(o.t, err, "Error funding Chainlink nodes") +} - var forwarders []common.Address - if o.OperatorForwarderFlow { - var operators []common.Address - operators, forwarders, _ = actions.DeployForwarderContracts( - o.t, o.seth, common.HexToAddress(linkContract.Address()), len(o.workerNodes), - ) - require.Equal(o.t, len(o.workerNodes), len(operators), "Number of operators should match number of nodes") - require.Equal(o.t, len(o.workerNodes), len(forwarders), "Number of authorized forwarders should match number of nodes") - forwarderNodesAddresses, err := actions.ChainlinkNodeAddresses(o.workerNodes) - require.NoError(o.t, err, "Retrieving on-chain wallet addresses for chainlink nodes shouldn't fail") - for i := range o.workerNodes { - actions.AcceptAuthorizedReceiversOperator( - o.t, o.log, o.seth, operators[i], forwarders[i], []common.Address{forwarderNodesAddresses[i]}) - require.NoError(o.t, err, "Accepting Authorize Receivers on Operator shouldn't fail") - actions.TrackForwarder(o.t, o.seth, forwarders[i], o.workerNodes[i]) - } - } else if o.OCRVersion == "1" { - if o.OperatorForwarderFlow { - o.ocrV1Instances, err = actions.DeployOCRContractsForwarderFlow( - o.log, - o.seth, - o.Config.GetActiveOCRConfig(), - common.HexToAddress(linkContract.Address()), - contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(o.workerNodes), - forwarders, - ) - require.NoError(o.t, err, "Error deploying OCR Forwarder contracts") - } else { - o.ocrV1Instances, err = actions.SetupOCRv1Contracts( - o.log, - sethClient, - o.Config.GetActiveOCRConfig(), - common.HexToAddress(linkContract.Address()), - contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(o.workerNodes), - ) - require.NoError(o.t, err) - } - } else if o.OCRVersion == "2" { - var transmitters []string +// deployForwarderContracts deploys forwarder contracts if OperatorForwarderFlow is enabled. +func (o *OCRSoakTest) deployForwarderContracts() (operators []common.Address, forwarders []common.Address) { + operators, forwarders, _ = actions.DeployForwarderContracts( + o.t, o.sethClient, common.HexToAddress(o.linkContract.Address()), len(o.workerNodes), + ) + require.Equal(o.t, len(o.workerNodes), len(operators), "Number of operators should match number of nodes") + require.Equal(o.t, len(o.workerNodes), len(forwarders), "Number of authorized forwarders should match number of nodes") + + forwarderNodesAddresses, err := actions.ChainlinkNodeAddresses(o.workerNodes) + require.NoError(o.t, err, "Retrieving on-chain wallet addresses for chainlink nodes shouldn't fail") + for i := range o.workerNodes { + actions.AcceptAuthorizedReceiversOperator(o.t, o.log, o.sethClient, operators[i], forwarders[i], []common.Address{forwarderNodesAddresses[i]}) + require.NoError(o.t, err, "Accepting Authorize Receivers on Operator shouldn't fail") + actions.TrackForwarder(o.t, o.sethClient, forwarders[i], o.workerNodes[i]) + } + return operators, forwarders +} - if o.OperatorForwarderFlow { - for _, forwarder := range forwarders { - transmitters = append(transmitters, forwarder.Hex()) - } - } else { - for _, node := range o.workerNodes { - nodeAddress, err := node.PrimaryEthAddress() - require.NoError(o.t, err, "Error getting node's primary ETH address") - transmitters = append(transmitters, nodeAddress) - } - } +// setupOCRContracts deploys and configures OCR contracts based on the version and forwarder flow. +func (o *OCRSoakTest) setupOCRContracts(ocrTestConfig tt.OcrTestConfig, forwarders []common.Address) { + if o.OCRVersion == "1" { + o.setupOCRv1Contracts(forwarders) + } else if o.OCRVersion == "2" { + o.setupOCRv2Contracts(ocrTestConfig, forwarders) + } +} - ocrOffchainOptions := contracts.DefaultOffChainAggregatorOptions() - o.ocrV2Instances, err = actions.SetupOCRv2Contracts( +// setupOCRv1Contracts deploys and configures OCRv1 contracts based on the forwarder flow. +func (o *OCRSoakTest) setupOCRv1Contracts(forwarders []common.Address) { + var err error + if o.OperatorForwarderFlow { + o.ocrV1Instances, err = actions.DeployOCRContractsForwarderFlow( o.log, - o.seth, - ocrTestConfig.GetActiveOCRConfig(), - common.HexToAddress(linkContract.Address()), - transmitters, - ocrOffchainOptions, + o.sethClient, + o.Config.GetActiveOCRConfig(), + common.HexToAddress(o.linkContract.Address()), + contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(o.workerNodes), + forwarders, ) - require.NoError(o.t, err, "Error deploying OCRv2 contracts") - - if !ocrTestConfig.GetActiveOCRConfig().UseExistingOffChainAggregatorsContracts() || (ocrTestConfig.GetActiveOCRConfig().UseExistingOffChainAggregatorsContracts() && ocrTestConfig.GetActiveOCRConfig().ConfigureExistingOffChainAggregatorsContracts()) { - contractConfig, err := actions.BuildMedianOCR2Config(o.workerNodes, ocrOffchainOptions) - require.NoError(o.t, err, "Error building median config") - err = actions.ConfigureOCRv2AggregatorContracts(contractConfig, o.ocrV2Instances) - require.NoError(o.t, err, "Error configuring OCRv2 aggregator contracts") - } + require.NoError(o.t, err, "Error deploying OCR Forwarder contracts") + o.createJobsWithForwarder() + } else { + o.ocrV1Instances, err = actions.SetupOCRv1Contracts( + o.log, + o.sethClient, + o.Config.GetActiveOCRConfig(), + common.HexToAddress(o.linkContract.Address()), + contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(o.workerNodes), + ) + require.NoError(o.t, err, "Error setting up OCRv1 contracts") + err = o.createOCRv1Jobs() + require.NoError(o.t, err, "Error creating OCR jobs") } - if o.OCRVersion == "1" { - for _, ocrInstance := range o.ocrV1Instances { - o.ocrV1InstanceMap[ocrInstance.Address()] = ocrInstance + o.storeOCRInstancesV1() +} + +// setupOCRv2Contracts sets up and configures OCRv2 contracts. +func (o *OCRSoakTest) setupOCRv2Contracts(ocrTestConfig tt.OcrTestConfig, forwarders []common.Address) { + var err error + var transmitters []string + if o.OperatorForwarderFlow { + for _, forwarder := range forwarders { + transmitters = append(transmitters, forwarder.Hex()) } - } else if o.OCRVersion == "2" { - for _, ocrInstance := range o.ocrV2Instances { - o.ocrV2InstanceMap[ocrInstance.Address()] = ocrInstance + } else { + for _, node := range o.workerNodes { + nodeAddress, err := node.PrimaryEthAddress() + require.NoError(o.t, err, "Error getting node's primary ETH address") + transmitters = append(transmitters, nodeAddress) } } - o.log.Info().Msg("OCR Soak Test Setup Complete") + ocrOffchainOptions := contracts.DefaultOffChainAggregatorOptions() + o.ocrV2Instances, err = actions.SetupOCRv2Contracts( + o.log, o.sethClient, ocrTestConfig.GetActiveOCRConfig(), common.HexToAddress(o.linkContract.Address()), transmitters, ocrOffchainOptions, + ) + require.NoError(o.t, err, "Error deploying OCRv2 contracts") + err = o.createOCRv2Jobs() + require.NoError(o.t, err, "Error creating OCR jobs") + if !ocrTestConfig.GetActiveOCRConfig().UseExistingOffChainAggregatorsContracts() || (ocrTestConfig.GetActiveOCRConfig().UseExistingOffChainAggregatorsContracts() && ocrTestConfig.GetActiveOCRConfig().ConfigureExistingOffChainAggregatorsContracts()) { + contractConfig, err := actions.BuildMedianOCR2Config(o.workerNodes, ocrOffchainOptions) + require.NoError(o.t, err, "Error building median config") + err = actions.ConfigureOCRv2AggregatorContracts(contractConfig, o.ocrV2Instances) + require.NoError(o.t, err, "Error configuring OCRv2 aggregator contracts") + } + o.storeOCRInstancesV2() +} + +// storeOCRInstancesV1 stores OCRv1 contract instances by their addresses. +func (o *OCRSoakTest) storeOCRInstancesV1() { + for _, ocrInstance := range o.ocrV1Instances { + o.ocrV1InstanceMap[ocrInstance.Address()] = ocrInstance + } +} + +// storeOCRInstancesV2 stores OCRv2 contract instances by their addresses. +func (o *OCRSoakTest) storeOCRInstancesV2() { + for _, ocrInstance := range o.ocrV2Instances { + o.ocrV2InstanceMap[ocrInstance.Address()] = ocrInstance + } } // Run starts the OCR soak test @@ -389,36 +436,52 @@ func (o *OCRSoakTest) Run() { require.NoError(o.t, err, "Error getting config") ctx, cancel := context.WithTimeout(testcontext.Get(o.t), time.Second*5) - latestBlockNum, err := o.seth.Client.BlockNumber(ctx) + latestBlockNum, err := o.sethClient.Client.BlockNumber(ctx) cancel() require.NoError(o.t, err, "Error getting current block number") o.startingBlockNum = latestBlockNum - startingValue := 5 - if o.OperatorForwarderFlow { - actions.CreateOCRJobsWithForwarder(o.t, o.ocrV1Instances, o.bootstrapNode, o.workerNodes, startingValue, o.mockServer, o.seth.ChainID) - } else if o.OCRVersion == "1" { - ctx, cancel := context.WithTimeout(testcontext.Get(o.t), time.Second*5) - chainId, err := o.seth.Client.ChainID(ctx) - cancel() - require.NoError(o.t, err, "Error getting chain ID") - err = actions.CreateOCRJobs(o.ocrV1Instances, o.bootstrapNode, o.workerNodes, startingValue, o.mockServer, chainId.String()) - require.NoError(o.t, err, "Error creating OCR jobs") - } else if o.OCRVersion == "2" { - err := actions.CreateOCRv2Jobs(o.ocrV2Instances, o.bootstrapNode, o.workerNodes, o.mockServer, startingValue, o.seth.ChainID, o.OperatorForwarderFlow) - require.NoError(o.t, err, "Error creating OCR jobs") - } - o.log.Info(). Str("Test Duration", o.Config.GetActiveOCRConfig().Common.TestDuration.Duration.Truncate(time.Second).String()). Int("Number of OCR Contracts", *config.GetActiveOCRConfig().Common.NumberOfContracts). Str("OCR Version", o.OCRVersion). Msg("Starting OCR Soak Test") - o.testLoop(o.Config.GetActiveOCRConfig().Common.TestDuration.Duration, startingValue) + o.testLoop(o.Config.GetActiveOCRConfig().Common.TestDuration.Duration, o.startingValue) o.complete() } +// createJobsWithForwarder creates OCR jobs with the forwarder setup. +func (o *OCRSoakTest) createJobsWithForwarder() { + actions.CreateOCRJobsWithForwarder(o.t, o.ocrV1Instances, o.bootstrapNode, o.workerNodes, o.startingValue, o.mockServer, o.sethClient.ChainID) +} + +// createOCRv1Jobs creates OCRv1 jobs. +func (o *OCRSoakTest) createOCRv1Jobs() error { + ctx, cancel := context.WithTimeout(testcontext.Get(o.t), time.Second*5) + defer cancel() + + chainId, err := o.sethClient.Client.ChainID(ctx) + if err != nil { + return fmt.Errorf("error getting chain ID: %w", err) + } + + err = actions.CreateOCRJobs(o.ocrV1Instances, o.bootstrapNode, o.workerNodes, o.startingValue, o.mockServer, chainId.String()) + if err != nil { + return fmt.Errorf("error creating OCRv1 jobs: %w", err) + } + return nil +} + +// createOCRv2Jobs creates OCRv2 jobs. +func (o *OCRSoakTest) createOCRv2Jobs() error { + err := actions.CreateOCRv2Jobs(o.ocrV2Instances, o.bootstrapNode, o.workerNodes, o.mockServer, o.startingValue, o.sethClient.ChainID, o.OperatorForwarderFlow, o.log) + if err != nil { + return fmt.Errorf("error creating OCRv2 jobs: %w", err) + } + return nil +} + // Networks returns the networks that the test is running on func (o *OCRSoakTest) TearDownVals(t *testing.T) ( *testing.T, @@ -428,7 +491,7 @@ func (o *OCRSoakTest) TearDownVals(t *testing.T) ( reportModel.TestReporter, reportModel.GrafanaURLProvider, ) { - return t, o.seth, o.namespace, append(o.workerNodes, o.bootstrapNode), &o.TestReporter, o.Config + return t, o.sethClient, o.namespace, append(o.workerNodes, o.bootstrapNode), &o.TestReporter, o.Config } // ********************* @@ -542,7 +605,7 @@ func (o *OCRSoakTest) LoadState() error { if testState.OCRVersion == "1" { o.ocrV1Instances = make([]contracts.OffchainAggregator, len(testState.OCRContractAddresses)) for i, addr := range testState.OCRContractAddresses { - instance, err := contracts.LoadOffChainAggregator(o.log, o.seth, common.HexToAddress(addr)) + instance, err := contracts.LoadOffChainAggregator(o.log, o.sethClient, common.HexToAddress(addr)) if err != nil { return fmt.Errorf("failed to instantiate OCR instance: %w", err) } @@ -551,7 +614,7 @@ func (o *OCRSoakTest) LoadState() error { } else if testState.OCRVersion == "2" { o.ocrV2Instances = make([]contracts.OffchainAggregatorV2, len(testState.OCRContractAddresses)) for i, addr := range testState.OCRContractAddresses { - instance, err := contracts.LoadOffchainAggregatorV2(o.log, o.seth, common.HexToAddress(addr)) + instance, err := contracts.LoadOffchainAggregatorV2(o.log, o.sethClient, common.HexToAddress(addr)) if err != nil { return err } @@ -790,7 +853,7 @@ func (o *OCRSoakTest) startAnvilGasSpikeSimulation(network blockchain.EVMNetwork func (o *OCRSoakTest) startAnvilGasLimitSimulation(network blockchain.EVMNetwork, conf ctf_config.GasLimitSimulationConfig) { client := ctf_client.NewRPCClient(network.HTTPURLs[0], nil) - latestBlock, err := o.seth.Client.BlockByNumber(context.Background(), nil) + latestBlock, err := o.sethClient.Client.BlockByNumber(context.Background(), nil) require.NoError(o.t, err) newGasLimit := int64(math.Ceil(float64(latestBlock.GasUsed()) * conf.NextGasLimitPercentage)) o.log.Info(). @@ -916,7 +979,7 @@ func (o *OCRSoakTest) pollingOCREvents(ctx context.Context, wg *sync.WaitGroup, // Helper function to poll events and update eventCounter func (o *OCRSoakTest) fetchAndProcessEvents(eventCounter *int, expectedEvents int, processedBlockNum *uint64) { - latestBlock, err := o.seth.Client.BlockNumber(context.Background()) + latestBlock, err := o.sethClient.Client.BlockNumber(context.Background()) if err != nil { o.log.Error().Err(err).Msg("Error getting latest block number") return @@ -949,7 +1012,7 @@ func (o *OCRSoakTest) fetchAndProcessEvents(eventCounter *int, expectedEvents in Uint64("To Block", latestBlock). Msg("Fetching logs for the specified range") - logs, err := o.seth.Client.FilterLogs(context.Background(), o.filterQuery) + logs, err := o.sethClient.Client.FilterLogs(context.Background(), o.filterQuery) if err != nil { o.log.Error().Err(err).Msg("Error fetching logs") return @@ -1066,12 +1129,12 @@ func (o *OCRSoakTest) collectEvents() error { o.log.Info().Interface("Filter Query", o.filterQuery).Str("Timeout", timeout.String()).Msg("Retrieving on-chain events") ctx, cancel := context.WithTimeout(testcontext.Get(o.t), timeout) - contractEvents, err := o.seth.Client.FilterLogs(ctx, o.filterQuery) + contractEvents, err := o.sethClient.Client.FilterLogs(ctx, o.filterQuery) cancel() for err != nil { o.log.Info().Interface("Filter Query", o.filterQuery).Str("Timeout", timeout.String()).Msg("Retrieving on-chain events") ctx, cancel := context.WithTimeout(testcontext.Get(o.t), timeout) - contractEvents, err = o.seth.Client.FilterLogs(ctx, o.filterQuery) + contractEvents, err = o.sethClient.Client.FilterLogs(ctx, o.filterQuery) cancel() if err != nil { o.log.Warn().Interface("Filter Query", o.filterQuery).Str("Timeout", timeout.String()).Msg("Error collecting on-chain events, trying again")