From 83a36e9c58ddc924d79adda3c8327a47c1231d1f Mon Sep 17 00:00:00 2001 From: Makram Date: Fri, 6 Dec 2024 16:47:31 +0200 Subject: [PATCH] integration-tests/smoke/ccip: increase batch size (#15444) * integration-tests/smoke/ccip: increase batch size * bump balance of sender * bump cl-ccip to main * deployment/ccip/changeset: conform to ChangeSet iface * Revert "deployment/ccip/changeset: conform to ChangeSet iface" This reverts commit 449a77d1f5b07cc751c6f084827b7d53b8975254. * changes * split into separate tests * fix lint * rm comment * const * bump to a million --- .github/e2e-tests.yml | 13 - .github/integration-in-memory-tests.yml | 8 + deployment/ccip/changeset/test_helpers.go | 13 +- deployment/environment.go | 3 + deployment/environment/memory/chain.go | 6 +- deployment/environment/memory/environment.go | 1 + .../smoke/ccip/ccip_batching_test.go | 578 +++++++++--------- 7 files changed, 312 insertions(+), 310 deletions(-) diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index a671c081c1a..3b394293378 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -948,19 +948,6 @@ runner-test-matrix: E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 E2E_JD_VERSION: 0.6.0 - - id: smoke/ccip/ccip_batching_test.go:* - path: integration-tests/smoke/ccip/ccip_batching_test.go - test_env_type: docker - runs_on: ubuntu-latest - triggers: - - PR E2E Core Tests - - Nightly E2E Tests - test_cmd: cd integration-tests/ && go test smoke/ccip/ccip_batching_test.go -timeout 12m -test.parallel=1 -count=1 -json - pyroscope_env: ci-smoke-ccipv1_6-evm-simulated - test_env_vars: - E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2,SIMULATED_3 - E2E_JD_VERSION: 0.6.0 - - id: smoke/ccip/ccip_token_transfer_test.go:* path: integration-tests/smoke/ccip/ccip_token_transfer_test.go test_env_type: docker diff --git a/.github/integration-in-memory-tests.yml b/.github/integration-in-memory-tests.yml index c5e0f088afe..4b4fd71258d 100644 --- a/.github/integration-in-memory-tests.yml +++ b/.github/integration-in-memory-tests.yml @@ -31,6 +31,14 @@ runner-test-matrix: triggers: - PR Integration CCIP Tests test_cmd: cd integration-tests/smoke/ccip && go test ccip_fee_boosting_test.go -timeout 12m -test.parallel=2 -count=1 -json + + - id: smoke/ccip/ccip_batching_test.go:* + path: integration-tests/smoke/ccip/ccip_batching_test.go + test_env_type: in-memory + runs_on: ubuntu-latest + triggers: + - PR Integration CCIP Tests + test_cmd: cd integration-tests/smoke/ccip && go test ccip_batching_test.go -timeout 12m -test.parallel=2 -count=1 -json - id: contracts/ccipreader_test.go:* path: integration-tests/contracts/ccipreader_test.go diff --git a/deployment/ccip/changeset/test_helpers.go b/deployment/ccip/changeset/test_helpers.go index eb719450a0a..1ee8b0d0e42 100644 --- a/deployment/ccip/changeset/test_helpers.go +++ b/deployment/ccip/changeset/test_helpers.go @@ -286,9 +286,15 @@ func NewMemoryEnvironmentWithJobsAndContracts(t *testing.T, lggr logger.Logger, TimelockMinDelay: big.NewInt(0), } } - var usdcChains []uint64 - if tCfg != nil && tCfg.IsUSDC { - usdcChains = allChains + var ( + usdcChains []uint64 + isMulticall3 bool + ) + if tCfg != nil { + if tCfg.IsUSDC { + usdcChains = allChains + } + isMulticall3 = tCfg.IsMultiCall3 } // Need to deploy prerequisites first so that we can form the USDC config // no proposals to be made, timelock can be passed as nil here @@ -303,6 +309,7 @@ func NewMemoryEnvironmentWithJobsAndContracts(t *testing.T, lggr logger.Logger, ChainSelectors: allChains, Opts: []PrerequisiteOpt{ WithUSDCChains(usdcChains), + WithMulticall3(isMulticall3), }, }, }, diff --git a/deployment/environment.go b/deployment/environment.go index bdf9fe6d5de..2de16a32cab 100644 --- a/deployment/environment.go +++ b/deployment/environment.go @@ -54,6 +54,9 @@ type Chain struct { // Note the Sign function can be abstract supporting a variety of key storage mechanisms (e.g. KMS etc). DeployerKey *bind.TransactOpts Confirm func(tx *types.Transaction) (uint64, error) + // Users are a set of keys that can be used to interact with the chain. + // These are distinct from the deployer key. + Users []*bind.TransactOpts } // Environment represents an instance of a deployed product diff --git a/deployment/environment/memory/chain.go b/deployment/environment/memory/chain.go index 58f71a83a8c..40a20a02416 100644 --- a/deployment/environment/memory/chain.go +++ b/deployment/environment/memory/chain.go @@ -9,12 +9,12 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient/simulated" - "github.com/ethereum/go-ethereum/params" "github.com/stretchr/testify/require" chainsel "github.com/smartcontractkit/chain-selectors" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" ) type EVMChain struct { @@ -66,7 +66,7 @@ func evmChain(t *testing.T, numUsers int) EVMChain { owner, err := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) require.NoError(t, err) genesis := types.GenesisAlloc{ - owner.From: {Balance: big.NewInt(0).Mul(big.NewInt(700000), big.NewInt(params.Ether))}} + owner.From: {Balance: assets.Ether(1_000_000).ToInt()}} // create a set of user keys var users []*bind.TransactOpts for j := 0; j < numUsers; j++ { @@ -75,7 +75,7 @@ func evmChain(t *testing.T, numUsers int) EVMChain { user, err := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) require.NoError(t, err) users = append(users, user) - genesis[user.From] = types.Account{Balance: big.NewInt(0).Mul(big.NewInt(700000), big.NewInt(params.Ether))} + genesis[user.From] = types.Account{Balance: assets.Ether(1_000_000).ToInt()} } // there have to be enough initial funds on each chain to allocate for all the nodes that share the given chain in the test backend := simulated.NewBackend(genesis, simulated.WithBlockGasLimit(50000000)) diff --git a/deployment/environment/memory/environment.go b/deployment/environment/memory/environment.go index f4692998d34..e8f0c265101 100644 --- a/deployment/environment/memory/environment.go +++ b/deployment/environment/memory/environment.go @@ -94,6 +94,7 @@ func generateMemoryChain(t *testing.T, inputs map[uint64]EVMChain) map[uint64]de return receipt.BlockNumber.Uint64(), nil } }, + Users: chain.Users, } } return chains diff --git a/integration-tests/smoke/ccip/ccip_batching_test.go b/integration-tests/smoke/ccip/ccip_batching_test.go index c801e899585..8c3615fbb20 100644 --- a/integration-tests/smoke/ccip/ccip_batching_test.go +++ b/integration-tests/smoke/ccip/ccip_batching_test.go @@ -9,20 +9,15 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - gethtypes "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/require" "golang.org/x/exp/maps" - chainsel "github.com/smartcontractkit/chain-selectors" - "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" "github.com/smartcontractkit/chainlink-common/pkg/merklemulti" + "github.com/smartcontractkit/chainlink/deployment/environment/memory" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" - testsetups "github.com/smartcontractkit/chainlink/integration-tests/testsetups/ccip" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/offramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/onramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" @@ -30,16 +25,33 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/logger" ) -func Test_CCIPBatching(t *testing.T) { +const ( + numMessages = 40 +) + +type batchTestSetup struct { + e changeset.DeployedEnv + state changeset.CCIPOnChainState + sourceChain1 uint64 + sourceChain2 uint64 + destChain uint64 +} + +func newBatchTestSetup(t *testing.T) batchTestSetup { // Setup 3 chains, with 2 lanes going to the dest. - lggr := logger.TestLogger(t) - ctx := changeset.Context(t) - // Will load 3 chains when specified by the overrides.toml or env vars (E2E_TEST_SELECTED_NETWORK). - // See e2e-tests.yml. - e, _, _ := testsetups.NewLocalDevEnvironmentWithDefaultPrice(t, lggr, &changeset.TestConfigs{ - IsUSDC: false, - IsMultiCall3: true, // needed for this test - }) + e := changeset.NewMemoryEnvironmentWithJobsAndContracts( + t, + logger.TestLogger(t), + memory.MemoryEnvironmentConfig{ + Chains: 3, + Nodes: 4, + Bootstraps: 1, + NumOfUsersPerChain: 2, + }, + &changeset.TestConfigs{ + IsMultiCall3: true, + }, + ) state, err := changeset.LoadOnchainState(e.Env) require.NoError(t, err) @@ -61,251 +73,260 @@ func Test_CCIPBatching(t *testing.T) { require.NoError(t, changeset.AddLaneWithDefaultPricesAndFeeQuoterConfig(e.Env, state, sourceChain1, destChain, false)) require.NoError(t, changeset.AddLaneWithDefaultPricesAndFeeQuoterConfig(e.Env, state, sourceChain2, destChain, false)) - const ( - numMessages = 5 - ) + return batchTestSetup{e, state, sourceChain1, sourceChain2, destChain} +} + +func Test_CCIPBatching_MaxBatchSizeEVM(t *testing.T) { + t.Parallel() + + ctx := changeset.Context(t) + setup := newBatchTestSetup(t) + sourceChain1, sourceChain2, destChain, e, state := setup.sourceChain1, setup.sourceChain2, setup.destChain, setup.e, setup.state + var ( startSeqNum = map[uint64]ccipocr3.SeqNum{ sourceChain1: 1, sourceChain2: 1, } - ) - - t.Run("batch data only messages from single source", func(t *testing.T) { - var ( - sourceChain = sourceChain1 - ) - err := sendMessages( - ctx, - t, - e.Env.Chains[sourceChain], + sourceChain = sourceChain1 + transactors = []*bind.TransactOpts{ e.Env.Chains[sourceChain].DeployerKey, - state.Chains[sourceChain].OnRamp, - state.Chains[sourceChain].Router, - state.Chains[sourceChain].Multicall3, - destChain, - numMessages, - common.LeftPadBytes(state.Chains[destChain].Receiver.Address().Bytes(), 32), - ) - require.NoError(t, err) - - _, err = changeset.ConfirmCommitWithExpectedSeqNumRange( - t, - e.Env.Chains[sourceChain], - e.Env.Chains[destChain], - state.Chains[destChain].OffRamp, - nil, - ccipocr3.NewSeqNumRange(startSeqNum[sourceChain], startSeqNum[sourceChain]+numMessages-1), - true, - ) - require.NoErrorf(t, err, "failed to confirm commit from chain %d", sourceChain) - - states, err := changeset.ConfirmExecWithSeqNrs( - t, - e.Env.Chains[sourceChain], - e.Env.Chains[destChain], - state.Chains[destChain].OffRamp, - nil, - genSeqNrRange(startSeqNum[sourceChain], startSeqNum[sourceChain]+numMessages-1), - ) - require.NoError(t, err) - // assert that all states are successful - for _, state := range states { - require.Equal(t, changeset.EXECUTION_STATE_SUCCESS, state) + e.Env.Chains[sourceChain].Users[0], } + errs = make(chan error, len(transactors)) + ) - startSeqNum[sourceChain] = startSeqNum[sourceChain] + numMessages - }) - - t.Run("batch data only messages from multiple sources", func(t *testing.T) { - t.Skipf("skipping - failing consistently in CI") - var ( - wg sync.WaitGroup - sourceChains = []uint64{sourceChain1, sourceChain2} - errs = make(chan error, len(sourceChains)) - ) - - for _, srcChain := range sourceChains { - wg.Add(1) - go sendMessagesAsync( + for _, transactor := range transactors { + go func() { + err := sendMessages( ctx, t, - e, - state, - srcChain, + e.Env.Chains[sourceChain], + transactor, + state.Chains[sourceChain].OnRamp, + state.Chains[sourceChain].Router, + state.Chains[sourceChain].Multicall3, destChain, - numMessages, - &wg, - errs, + merklemulti.MaxNumberTreeLeaves/2, + common.LeftPadBytes(state.Chains[destChain].Receiver.Address().Bytes(), 32), ) - } - - wg.Wait() - - var i int - for i < len(sourceChains) { - select { - case err := <-errs: - require.NoError(t, err) - i++ - case <-ctx.Done(): - require.FailNow(t, "didn't get all errors before test context was done") - } - } + t.Log("sendMessages error:", err, ", writing to channel") + errs <- err + t.Log("sent error to channel") + }() + } - // confirm the commit reports - outputErrs := make(chan outputErr[*offramp.OffRampCommitReportAccepted], len(sourceChains)) - for _, srcChain := range sourceChains { - wg.Add(1) - go assertCommitReportsAsync( - t, - e, - state, - srcChain, - destChain, - startSeqNum[srcChain], - startSeqNum[srcChain]+ccipocr3.SeqNum(numMessages)-1, - &wg, - outputErrs, - ) + var i = 0 + for i < len(transactors) { + select { + case err := <-errs: + require.NoError(t, err) + i++ + case <-ctx.Done(): + require.FailNow(t, "didn't get all errors before test context was done") } + } - t.Log("waiting for commit report") - wg.Wait() - - i = 0 - var reports []*offramp.OffRampCommitReportAccepted - for i < len(sourceChains) { - select { - case outputErr := <-outputErrs: - require.NoError(t, outputErr.err) - reports = append(reports, outputErr.output) - i++ - case <-ctx.Done(): - require.FailNow(t, "didn't get all commit reports before test context was done") - } - } + _, err := changeset.ConfirmCommitWithExpectedSeqNumRange( + t, + e.Env.Chains[sourceChain], + e.Env.Chains[destChain], + state.Chains[destChain].OffRamp, + nil, // startBlock + ccipocr3.NewSeqNumRange( + startSeqNum[sourceChain], + startSeqNum[sourceChain]+ccipocr3.SeqNum(merklemulti.MaxNumberTreeLeaves)-1, + ), + true, + ) + require.NoErrorf(t, err, "failed to confirm commit from chain %d", sourceChain) +} - // the reports should be the same for both, since both roots should be batched within - // that one report. - require.Lenf(t, reports, len(sourceChains), "expected %d commit reports", len(sourceChains)) - require.NotNil(t, reports[0], "commit report should not be nil") - require.NotNil(t, reports[1], "commit report should not be nil") - // TODO: this assertion is failing, despite messages being sent at the same time. - // require.Equal(t, reports[0], reports[1], "commit reports should be the same") - - // confirm execution - execErrs := make(chan outputErr[map[uint64]int], len(sourceChains)) - for _, srcChain := range sourceChains { - wg.Add(1) - go assertExecAsync( - t, - e, - state, - srcChain, - destChain, - genSeqNrRange(startSeqNum[srcChain], startSeqNum[srcChain]+ccipocr3.SeqNum(numMessages)-1), - &wg, - execErrs, - ) - } +func Test_CCIPBatching_MultiSource(t *testing.T) { + // t.Skip("Exec not working, boosting not working correctly") - t.Log("waiting for exec reports") - wg.Wait() - - i = 0 - var execStates []map[uint64]int - for i < len(sourceChains) { - select { - case outputErr := <-execErrs: - require.NoError(t, outputErr.err) - execStates = append(execStates, outputErr.output) - i++ - case <-ctx.Done(): - require.FailNow(t, "didn't get all exec reports before test context was done") - } - } + t.Parallel() - // assert that all states are successful - for _, states := range execStates { - for _, state := range states { - require.Equal(t, changeset.EXECUTION_STATE_SUCCESS, state) - } - } + // Setup 3 chains, with 2 lanes going to the dest. + ctx := changeset.Context(t) + setup := newBatchTestSetup(t) + sourceChain1, sourceChain2, destChain, e, state := setup.sourceChain1, setup.sourceChain2, setup.destChain, setup.e, setup.state - // update the start and end seq nums - for _, srcChain := range sourceChains { - startSeqNum[srcChain] = startSeqNum[srcChain] + numMessages + var ( + wg sync.WaitGroup + sourceChains = []uint64{sourceChain1, sourceChain2} + errs = make(chan error, len(sourceChains)) + startSeqNum = map[uint64]ccipocr3.SeqNum{ + sourceChain1: 1, + sourceChain2: 1, } - }) - - t.Run("max evm batch size", func(t *testing.T) { - t.Skipf("This test is flaky, skipping until the issue related to fee calculation is resolved") - var ( - sourceChain = sourceChain1 - otherSender = mustNewTransactor(t, e.Env.Chains[sourceChain]) - transactors = []*bind.TransactOpts{ - e.Env.Chains[sourceChain].DeployerKey, - otherSender, - } - errs = make(chan error, len(transactors)) - ) + ) - // transfer some eth to the other sender from the DeployerKey - sendEth( + for _, srcChain := range sourceChains { + wg.Add(1) + go sendMessagesAsync( ctx, t, - e.Env.Chains[sourceChain], - e.Env.Chains[sourceChain].DeployerKey, - otherSender.From, - assets.Ether(20).ToInt(), + e, + state, + srcChain, + destChain, + numMessages, + &wg, + errs, ) + } - for _, transactor := range transactors { - go func() { - err := sendMessages( - ctx, - t, - e.Env.Chains[sourceChain], - transactor, - state.Chains[sourceChain].OnRamp, - state.Chains[sourceChain].Router, - state.Chains[sourceChain].Multicall3, - destChain, - merklemulti.MaxNumberTreeLeaves/2, - common.LeftPadBytes(state.Chains[destChain].Receiver.Address().Bytes(), 32), - ) - t.Log("sendMessages error:", err, ", writing to channel") - errs <- err - t.Log("sent error to channel") - }() + wg.Wait() + + var i int + for i < len(sourceChains) { + select { + case err := <-errs: + require.NoError(t, err) + i++ + case <-ctx.Done(): + require.FailNow(t, "didn't get all errors before test context was done") } + } - var i = 0 - for i < len(transactors) { - select { - case err := <-errs: - require.NoError(t, err) - i++ - case <-ctx.Done(): - require.FailNow(t, "didn't get all errors before test context was done") - } + // confirm the commit reports + outputErrs := make(chan outputErr[*offramp.OffRampCommitReportAccepted], len(sourceChains)) + for _, srcChain := range sourceChains { + wg.Add(1) + go assertCommitReportsAsync( + t, + e, + state, + srcChain, + destChain, + startSeqNum[srcChain], + startSeqNum[srcChain]+ccipocr3.SeqNum(numMessages)-1, + &wg, + outputErrs, + ) + } + + t.Log("waiting for commit report") + wg.Wait() + + i = 0 + var reports []*offramp.OffRampCommitReportAccepted + for i < len(sourceChains) { + select { + case outputErr := <-outputErrs: + require.NoError(t, outputErr.err) + reports = append(reports, outputErr.output) + i++ + case <-ctx.Done(): + require.FailNow(t, "didn't get all commit reports before test context was done") } + } - _, err = changeset.ConfirmCommitWithExpectedSeqNumRange( + // the reports should be the same for both, since both roots should be batched within + // that one report. + require.Lenf(t, reports, len(sourceChains), "expected %d commit reports", len(sourceChains)) + require.NotNil(t, reports[0], "commit report should not be nil") + require.NotNil(t, reports[1], "commit report should not be nil") + // TODO: this assertion is failing, despite messages being sent at the same time. + // require.Equal(t, reports[0], reports[1], "commit reports should be the same") + + // confirm execution + execErrs := make(chan outputErr[map[uint64]int], len(sourceChains)) + for _, srcChain := range sourceChains { + wg.Add(1) + go assertExecAsync( t, - e.Env.Chains[sourceChain], - e.Env.Chains[destChain], - state.Chains[destChain].OffRamp, - nil, // startBlock - ccipocr3.NewSeqNumRange( - startSeqNum[sourceChain], - startSeqNum[sourceChain]+ccipocr3.SeqNum(merklemulti.MaxNumberTreeLeaves)-1, - ), - true, + e, + state, + srcChain, + destChain, + genSeqNrRange(startSeqNum[srcChain], startSeqNum[srcChain]+ccipocr3.SeqNum(numMessages)-1), + &wg, + execErrs, ) - require.NoErrorf(t, err, "failed to confirm commit from chain %d", sourceChain) - }) + } + + t.Log("waiting for exec reports") + wg.Wait() + + i = 0 + var execStates []map[uint64]int + for i < len(sourceChains) { + select { + case outputErr := <-execErrs: + require.NoError(t, outputErr.err) + execStates = append(execStates, outputErr.output) + i++ + case <-ctx.Done(): + require.FailNow(t, "didn't get all exec reports before test context was done") + } + } + + // assert that all states are successful + for _, states := range execStates { + for _, state := range states { + require.Equal(t, changeset.EXECUTION_STATE_SUCCESS, state) + } + } +} + +func Test_CCIPBatching_SingleSource(t *testing.T) { + t.Parallel() + + // Setup 3 chains, with 2 lanes going to the dest. + ctx := changeset.Context(t) + setup := newBatchTestSetup(t) + sourceChain1, sourceChain2, destChain, e, state := setup.sourceChain1, setup.sourceChain2, setup.destChain, setup.e, setup.state + + var ( + startSeqNum = map[uint64]ccipocr3.SeqNum{ + sourceChain1: 1, + sourceChain2: 1, + } + ) + + var ( + sourceChain = sourceChain1 + ) + err := sendMessages( + ctx, + t, + e.Env.Chains[sourceChain], + e.Env.Chains[sourceChain].DeployerKey, + state.Chains[sourceChain].OnRamp, + state.Chains[sourceChain].Router, + state.Chains[sourceChain].Multicall3, + destChain, + numMessages, + common.LeftPadBytes(state.Chains[destChain].Receiver.Address().Bytes(), 32), + ) + require.NoError(t, err) + + _, err = changeset.ConfirmCommitWithExpectedSeqNumRange( + t, + e.Env.Chains[sourceChain], + e.Env.Chains[destChain], + state.Chains[destChain].OffRamp, + nil, + ccipocr3.NewSeqNumRange(startSeqNum[sourceChain], startSeqNum[sourceChain]+numMessages-1), + true, + ) + require.NoErrorf(t, err, "failed to confirm commit from chain %d", sourceChain) + + states, err := changeset.ConfirmExecWithSeqNrs( + t, + e.Env.Chains[sourceChain], + e.Env.Chains[destChain], + state.Chains[destChain].OffRamp, + nil, + genSeqNrRange(startSeqNum[sourceChain], startSeqNum[sourceChain]+numMessages-1), + ) + require.NoError(t, err) + // assert that all states are successful + for _, state := range states { + require.Equal(t, changeset.EXECUTION_STATE_SUCCESS, state) + } } type outputErr[T any] struct { @@ -373,18 +394,34 @@ func sendMessagesAsync( out chan<- error, ) { defer wg.Done() - err := sendMessages( - ctx, - t, - e.Env.Chains[sourceChainSelector], - e.Env.Chains[sourceChainSelector].DeployerKey, - state.Chains[sourceChainSelector].OnRamp, - state.Chains[sourceChainSelector].Router, - state.Chains[sourceChainSelector].Multicall3, - destChainSelector, - numMessages, - common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), + var err error + + const ( + numRetries = 3 ) + + // we retry a bunch of times just in case there is a race b/w the prices being + // posted and the messages being sent. + for i := 0; i < numRetries; i++ { + err = sendMessages( + ctx, + t, + e.Env.Chains[sourceChainSelector], + e.Env.Chains[sourceChainSelector].DeployerKey, + state.Chains[sourceChainSelector].OnRamp, + state.Chains[sourceChainSelector].Router, + state.Chains[sourceChainSelector].Multicall3, + destChainSelector, + numMessages, + common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), + ) + if err == nil { + break + } + + t.Log("sendMessagesAsync error is non-nil:", err, ", retrying") + } + t.Log("sendMessagesAsync error:", err, ", writing to channel") out <- err } @@ -412,8 +449,13 @@ func sendMessages( return fmt.Errorf("generate messages: %w", err) } + currBalance, err := sourceChain.Client.BalanceAt(ctx, sourceTransactOpts.From, nil) + if err != nil { + return fmt.Errorf("get balance: %w", err) + } + // Send the tx with the messages through the multicall - t.Logf("Sending %d messages with total value %s", numMessages, totalValue.String()) + t.Logf("Sending %d messages with total value %s, current balance: %s", numMessages, totalValue.String(), currBalance.String()) tx, err := sourceMulticall3.Aggregate3Value( &bind.TransactOpts{ From: sourceTransactOpts.From, @@ -434,7 +476,9 @@ func sendMessages( if err != nil { return fmt.Errorf("get message sent event: %w", err) } - defer iter.Close() + defer func() { + require.NoError(t, iter.Close()) + }() // there should be numMessages messages emitted for i := 0; i < numMessages; i++ { @@ -461,7 +505,7 @@ func genMessages( Data: []byte(fmt.Sprintf("hello world %d", i)), TokenAmounts: nil, FeeToken: common.HexToAddress("0x0"), - ExtraArgs: nil, + ExtraArgs: changeset.MakeEVMExtraArgsV2(50_000, false), } fee, err := sourceRouter.GetFee(&bind.CallOpts{Context: ctx}, destChainSelector, msg) @@ -495,51 +539,3 @@ func genSeqNrRange(start, end ccipocr3.SeqNum) []uint64 { } return seqNrs } - -func mustNewTransactor(t *testing.T, chain deployment.Chain) *bind.TransactOpts { - chainID, err := chainsel.GetChainIDFromSelector(chain.Selector) - require.NoError(t, err) - chainIDBig, ok := new(big.Int).SetString(chainID, 10) - require.True(t, ok, "evm chainID must be integral") - key, err := crypto.GenerateKey() - require.NoError(t, err) - transactor, err := bind.NewKeyedTransactorWithChainID(key, chainIDBig) - require.NoError(t, err) - return transactor -} - -func sendEth( - ctx context.Context, - t *testing.T, - chain deployment.Chain, - from *bind.TransactOpts, - to common.Address, - value *big.Int, -) { - balance, err := chain.Client.BalanceAt(ctx, from.From, nil) - require.NoError(t, err) - if balance.Cmp(value) < 0 { - t.Fatalf("insufficient balance: %s < %s", balance.String(), value.String()) - } - t.Logf("balance of from account %s: %s", from.From.String(), balance.String()) - - nonce, err := chain.Client.PendingNonceAt(ctx, from.From) - require.NoError(t, err) - gp, err := chain.Client.SuggestGasPrice(ctx) - require.NoError(t, err) - tx := gethtypes.NewTx(&gethtypes.LegacyTx{ - Nonce: nonce, - GasPrice: gp, - Gas: 21_000, - To: &to, - Value: value, - Data: nil, - }) - signedTx, err := from.Signer(from.From, tx) - require.NoError(t, err) - err = chain.Client.SendTransaction(ctx, signedTx) - require.NoError(t, err) - t.Log("sent funding tx:", signedTx.Hash().Hex()) - _, err = deployment.ConfirmIfNoError(chain, signedTx, err) - require.NoError(t, err) -}