From 2cd83fb5724b6f38853dc30d1eb76694f9dd1186 Mon Sep 17 00:00:00 2001 From: Anindita Ghosh <88458927+AnieeG@users.noreply.github.com> Date: Wed, 3 Jul 2024 18:11:13 -0700 Subject: [PATCH] Ccip-2578 - Enable slow and fast chain (#1122) --- .../ccip-tests/actions/ccip_helpers.go | 118 ++++----- .../ccip-tests/contracts/contract_models.go | 1 + .../ccip-tests/load/ccip_loadgen.go | 80 ++----- .../ccip-tests/load/ccip_multicall_loadgen.go | 25 +- .../ccip-tests/testconfig/README.md | 2 +- .../ccip1.4-stress-2024-06-27/tier-a.toml | 226 ++++++++++++++++++ .../ccip-tests/testreporters/ccip.go | 27 ++- .../ccip-tests/testsetups/ccip.go | 13 +- 8 files changed, 328 insertions(+), 164 deletions(-) diff --git a/integration-tests/ccip-tests/actions/ccip_helpers.go b/integration-tests/ccip-tests/actions/ccip_helpers.go index 2518e03342..e2b9b70871 100644 --- a/integration-tests/ccip-tests/actions/ccip_helpers.go +++ b/integration-tests/ccip-tests/actions/ccip_helpers.go @@ -397,16 +397,9 @@ func (ccipModule *CCIPCommon) ApproveTokens() error { } } - allowance, err := token.Allowance(ccipModule.ChainClient.GetDefaultWallet().Address(), ccipModule.Router.Address()) + err := token.Approve(ccipModule.ChainClient.GetDefaultWallet(), ccipModule.Router.Address(), ApprovedAmountToRouter) if err != nil { - return fmt.Errorf("failed to get allowance for token %s: %w", token.ContractAddress.Hex(), err) - } - if allowance.Cmp(ApprovedAmountToRouter) < 0 { - allowanceApprovalDelta := new(big.Int).Sub(ApprovedAmountToRouter, allowance) - err := token.Approve(ccipModule.ChainClient.GetDefaultWallet(), ccipModule.Router.Address(), allowanceApprovalDelta) - if err != nil { - return fmt.Errorf("failed to approve token %s: %w", token.ContractAddress.Hex(), err) - } + return fmt.Errorf("failed to approve token %s: %w", token.ContractAddress.Hex(), err) } if token.ContractAddress == ccipModule.FeeToken.EthAddress { isApproved = true @@ -1565,23 +1558,43 @@ func (sourceCCIP *SourceCCIPModule) UpdateBalance( func (sourceCCIP *SourceCCIPModule) AssertSendRequestedLogFinalized( lggr *zerolog.Logger, txHash common.Hash, + sendReqData []*contracts.SendReqEventData, prevEventAt time.Time, reqStats []*testreporters.RequestStat, ) (time.Time, uint64, error) { + if len(sendReqData) != len(reqStats) { + return time.Time{}, 0, fmt.Errorf("sendReqData and reqStats length mismatch") + } + var gasUsed uint64 + receipt, err := sourceCCIP.Common.ChainClient.GetTxReceipt(txHash) + if err == nil { + gasUsed = receipt.GasUsed + } lggr.Info().Msg("Waiting for CCIPSendRequested event log to be finalized") finalizedBlockNum, finalizedAt, err := sourceCCIP.Common.ChainClient.WaitForFinalizedTx(txHash) if err != nil || finalizedBlockNum == nil { - for _, stat := range reqStats { - stat.UpdateState(lggr, stat.SeqNum, testreporters.SourceLogFinalized, time.Since(prevEventAt), testreporters.Failure) + for i, stat := range reqStats { + stat.UpdateState(lggr, stat.SeqNum, testreporters.SourceLogFinalized, time.Since(prevEventAt), testreporters.Failure, &testreporters.TransactionStats{ + MsgID: fmt.Sprintf("0x%x", sendReqData[i].MessageId[:]), + Fee: sendReqData[i].Fee.String(), + NoOfTokensSent: sendReqData[i].NoOfTokens, + MessageBytesLength: int64(sendReqData[i].DataLength), + TxHash: txHash.Hex(), + }) } return time.Time{}, 0, fmt.Errorf("error waiting for CCIPSendRequested event log to be finalized - %w", err) } - for _, stat := range reqStats { + for i, stat := range reqStats { stat.UpdateState(lggr, stat.SeqNum, testreporters.SourceLogFinalized, finalizedAt.Sub(prevEventAt), testreporters.Success, - testreporters.TransactionStats{ - TxHash: txHash.Hex(), - FinalizedByBlock: finalizedBlockNum.String(), - FinalizedAt: finalizedAt.String(), + &testreporters.TransactionStats{ + MsgID: fmt.Sprintf("0x%x", sendReqData[i].MessageId[:]), + Fee: sendReqData[i].Fee.String(), + GasUsed: gasUsed, + NoOfTokensSent: sendReqData[i].NoOfTokens, + MessageBytesLength: int64(sendReqData[i].DataLength), + TxHash: txHash.Hex(), + FinalizedByBlock: finalizedBlockNum.String(), + FinalizedAt: finalizedAt.String(), }) } return finalizedAt, finalizedBlockNum.Uint64(), nil @@ -1640,13 +1653,7 @@ func (sourceCCIP *SourceCCIPModule) AssertEventCCIPSendRequested( Str("MsgID", fmt.Sprintf("0x%x", sendRequestedEvent.MessageId[:])). Logger()) // prevEventAt is the time when the message was successful, this should be same as the time when the event was emitted - reqStat[i].UpdateState(lggr, seqNum, testreporters.CCIPSendRe, 0, testreporters.Success, - testreporters.TransactionStats{ - MsgID: fmt.Sprintf("0x%x", sendRequestedEvent.MessageId[:]), - TxHash: "", - NoOfTokensSent: sendRequestedEvent.NoOfTokens, - MessageBytesLength: int64(sendRequestedEvent.DataLength), - }) + reqStat[i].UpdateState(lggr, seqNum, testreporters.CCIPSendRe, 0, testreporters.Success, nil) } var err error if len(sendRequestedEvents) == 0 { @@ -1660,7 +1667,10 @@ func (sourceCCIP *SourceCCIPModule) AssertEventCCIPSendRequested( if sourceCCIP.Common.IsConnectionRestoredRecently != nil && !sourceCCIP.Common.IsConnectionRestoredRecently.Load() { if resetTimer > 2 { for _, stat := range reqStat { - stat.UpdateState(lggr, 0, testreporters.CCIPSendRe, time.Since(prevEventAt), testreporters.Failure) + stat.UpdateState(lggr, 0, testreporters.CCIPSendRe, time.Since(prevEventAt), testreporters.Failure, + &testreporters.TransactionStats{ + TxHash: txHash, + }) } return nil, time.Now(), fmt.Errorf("possible RPC issue - CCIPSendRequested event is not found for tx %s", txHash) } @@ -1670,7 +1680,10 @@ func (sourceCCIP *SourceCCIPModule) AssertEventCCIPSendRequested( continue } for _, stat := range reqStat { - stat.UpdateState(lggr, 0, testreporters.CCIPSendRe, time.Since(prevEventAt), testreporters.Failure) + stat.UpdateState(lggr, 0, testreporters.CCIPSendRe, time.Since(prevEventAt), testreporters.Failure, + &testreporters.TransactionStats{ + TxHash: txHash, + }) } return nil, time.Now(), fmt.Errorf("CCIPSendRequested event is not found for tx %s", txHash) } @@ -2293,7 +2306,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged( lggr.Info().Int64("seqNum", int64(seqNum)).Uint8("ExecutionState", e.State).Msg("ExecutionStateChanged event received") reqStat.UpdateState(lggr, seqNum, testreporters.ExecStateChanged, receivedAt.Sub(timeNow), testreporters.Success, - testreporters.TransactionStats{ + &testreporters.TransactionStats{ TxHash: vLogs.TxHash.Hex(), MsgID: fmt.Sprintf("0x%x", e.MessageId[:]), GasUsed: gasUsed, @@ -2301,7 +2314,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged( ) return e.State, nil } - reqStat.UpdateState(lggr, seqNum, testreporters.ExecStateChanged, time.Since(timeNow), testreporters.Failure) + reqStat.UpdateState(lggr, seqNum, testreporters.ExecStateChanged, time.Since(timeNow), testreporters.Failure, nil) return e.State, fmt.Errorf("ExecutionStateChanged event state - expected %d actual - %d with data %x for seq num %v for lane %d-->%d", execState, testhelpers.MessageExecutionState(e.State), e.ReturnData, seqNum, destCCIP.SourceChainId, destCCIP.Common.ChainClient.GetChainID()) } @@ -2311,7 +2324,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged( if destCCIP.Common.IsConnectionRestoredRecently != nil && !destCCIP.Common.IsConnectionRestoredRecently.Load() { // if timer already has been reset 2 times we fail with warning if resetTimer > 2 { - reqStat.UpdateState(lggr, seqNum, testreporters.ExecStateChanged, time.Since(timeNow), testreporters.Failure) + reqStat.UpdateState(lggr, seqNum, testreporters.ExecStateChanged, time.Since(timeNow), testreporters.Failure, nil) return 0, fmt.Errorf("possible RPC issues - ExecutionStateChanged event not found for seq num %d for lane %d-->%d", seqNum, destCCIP.SourceChainId, destCCIP.Common.ChainClient.GetChainID()) } @@ -2320,7 +2333,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged( lggr.Info().Int("count of reset", resetTimer).Msg("Resetting timer to validate ExecutionStateChanged event") continue } - reqStat.UpdateState(lggr, seqNum, testreporters.ExecStateChanged, time.Since(timeNow), testreporters.Failure) + reqStat.UpdateState(lggr, seqNum, testreporters.ExecStateChanged, time.Since(timeNow), testreporters.Failure, nil) return 0, fmt.Errorf("ExecutionStateChanged event not found for seq num %d for lane %d-->%d", seqNum, destCCIP.SourceChainId, destCCIP.Common.ChainClient.GetChainID()) } @@ -2378,7 +2391,7 @@ func (destCCIP *DestCCIPModule) AssertEventReportAccepted( gasUsed = receipt.GasUsed } reqStat.UpdateState(lggr, seqNum, testreporters.Commit, totalTime, testreporters.Success, - testreporters.TransactionStats{ + &testreporters.TransactionStats{ GasUsed: gasUsed, TxHash: reportAccepted.Raw.TxHash.String(), CommitRoot: fmt.Sprintf("%x", reportAccepted.MerkleRoot), @@ -2390,7 +2403,7 @@ func (destCCIP *DestCCIPModule) AssertEventReportAccepted( // if there is connection issue reset the context : if destCCIP.Common.IsConnectionRestoredRecently != nil && !destCCIP.Common.IsConnectionRestoredRecently.Load() { if resetTimerCount > 2 { - reqStat.UpdateState(lggr, seqNum, testreporters.Commit, time.Since(prevEventAt), testreporters.Failure) + reqStat.UpdateState(lggr, seqNum, testreporters.Commit, time.Since(prevEventAt), testreporters.Failure, nil) return nil, time.Now().UTC(), fmt.Errorf("possible RPC issue - ReportAccepted is not found for seq num %d lane %d-->%d", seqNum, destCCIP.SourceChainId, destCCIP.Common.ChainClient.GetChainID()) } @@ -2399,7 +2412,7 @@ func (destCCIP *DestCCIPModule) AssertEventReportAccepted( lggr.Info().Int("count of reset", resetTimerCount).Msg("Resetting timer to validate ReportAccepted event") continue } - reqStat.UpdateState(lggr, seqNum, testreporters.Commit, time.Since(prevEventAt), testreporters.Failure) + reqStat.UpdateState(lggr, seqNum, testreporters.Commit, time.Since(prevEventAt), testreporters.Failure, nil) return nil, time.Now().UTC(), fmt.Errorf("ReportAccepted is not found for seq num %d lane %d-->%d", seqNum, destCCIP.SourceChainId, destCCIP.Common.ChainClient.GetChainID()) } @@ -2472,7 +2485,7 @@ func (destCCIP *DestCCIPModule) AssertReportBlessed( gasUsed = receipt.GasUsed } reqStat.UpdateState(lggr, seqNum, testreporters.ReportBlessed, receivedAt.Sub(prevEventAt), testreporters.Success, - testreporters.TransactionStats{ + &testreporters.TransactionStats{ GasUsed: gasUsed, TxHash: vLogs.TxHash.String(), CommitRoot: fmt.Sprintf("%x", CommitReport.MerkleRoot), @@ -2484,7 +2497,7 @@ func (destCCIP *DestCCIPModule) AssertReportBlessed( // if there is connection issue reset the context : if destCCIP.Common.IsConnectionRestoredRecently != nil && !destCCIP.Common.IsConnectionRestoredRecently.Load() { if resetTimerCount > 2 { - reqStat.UpdateState(lggr, seqNum, testreporters.ReportBlessed, time.Since(prevEventAt), testreporters.Failure) + reqStat.UpdateState(lggr, seqNum, testreporters.ReportBlessed, time.Since(prevEventAt), testreporters.Failure, nil) return time.Now().UTC(), fmt.Errorf("possible RPC issue - ReportBlessed is not found for interval min - %d max - %d lane %d-->%d", CommitReport.Min, CommitReport.Max, destCCIP.SourceChainId, destCCIP.Common.ChainClient.GetChainID()) } @@ -2493,7 +2506,7 @@ func (destCCIP *DestCCIPModule) AssertReportBlessed( lggr.Info().Int("count of reset", resetTimerCount).Msg("Resetting timer to validate ReportBlessed event") continue } - reqStat.UpdateState(lggr, seqNum, testreporters.ReportBlessed, time.Since(prevEventAt), testreporters.Failure) + reqStat.UpdateState(lggr, seqNum, testreporters.ReportBlessed, time.Since(prevEventAt), testreporters.Failure, nil) return time.Now().UTC(), fmt.Errorf("ReportBlessed is not found for interval min - %d max - %d lane %d-->%d", CommitReport.Min, CommitReport.Max, destCCIP.SourceChainId, destCCIP.Common.ChainClient.GetChainID()) } @@ -2532,7 +2545,7 @@ func (destCCIP *DestCCIPModule) AssertSeqNumberExecuted( // if there is connection issue reset the context : if destCCIP.Common.IsConnectionRestoredRecently != nil && !destCCIP.Common.IsConnectionRestoredRecently.Load() { if resetTimerCount > 2 { - reqStat.UpdateState(lggr, seqNumberBefore, testreporters.Commit, time.Since(timeNow), testreporters.Failure) + reqStat.UpdateState(lggr, seqNumberBefore, testreporters.Commit, time.Since(timeNow), testreporters.Failure, nil) return fmt.Errorf("possible RPC issue - sequence number is not increased for seq num %d lane %d-->%d", seqNumberBefore, destCCIP.SourceChainId, destCCIP.Common.ChainClient.GetChainID()) } @@ -2541,7 +2554,7 @@ func (destCCIP *DestCCIPModule) AssertSeqNumberExecuted( lggr.Info().Int("count of reset", resetTimerCount).Msg("Resetting timer to validate seqnumber increase in commit store") continue } - reqStat.UpdateState(lggr, seqNumberBefore, testreporters.Commit, time.Since(timeNow), testreporters.Failure) + reqStat.UpdateState(lggr, seqNumberBefore, testreporters.Commit, time.Since(timeNow), testreporters.Failure, nil) return fmt.Errorf("sequence number is not increased for seq num %d lane %d-->%d", seqNumberBefore, destCCIP.SourceChainId, destCCIP.Common.ChainClient.GetChainID()) } @@ -2740,7 +2753,7 @@ func (lane *CCIPLane) AddToSentReqs(txHash common.Hash, reqStats []*testreporter request, rcpt, err := CCIPRequestFromTxHash(txHash, lane.Source.Common.ChainClient) if err != nil { for _, stat := range reqStats { - stat.UpdateState(lane.Logger, 0, testreporters.TX, 0, testreporters.Failure) + stat.UpdateState(lane.Logger, 0, testreporters.TX, 0, testreporters.Failure, nil) } return rcpt, fmt.Errorf("could not get request from tx hash %s: %w", txHash.Hex(), err) } @@ -2772,7 +2785,7 @@ func (lane *CCIPLane) Multicall(noOfRequests int, multiSendAddr common.Address) return fmt.Errorf("failed getting the chain selector: %w", err) } var reqStats []*testreporters.RequestStat - var txstats []testreporters.TransactionStats + var txstats []*testreporters.TransactionStats for i := 1; i <= noOfRequests; i++ { // form the message for transfer msg := genericMsg @@ -2808,7 +2821,7 @@ func (lane *CCIPLane) Multicall(noOfRequests int, multiSendAddr common.Address) } } stat := testreporters.NewCCIPRequestStats(int64(lane.NumberOfReq+i), lane.SourceNetworkName, lane.DestNetworkName) - txstats = append(txstats, testreporters.TransactionStats{ + txstats = append(txstats, &testreporters.TransactionStats{ Fee: fee.String(), NoOfTokensSent: len(msg.TokenAmounts), MessageBytesLength: int64(len(msg.Data)), @@ -2829,7 +2842,7 @@ func (lane *CCIPLane) Multicall(noOfRequests int, multiSendAddr common.Address) if err != nil { // update the stats as failure for all the requests in the multicall tx for _, stat := range reqStats { - stat.UpdateState(lane.Logger, 0, testreporters.TX, 0, testreporters.Failure) + stat.UpdateState(lane.Logger, 0, testreporters.TX, 0, testreporters.Failure, nil) } return fmt.Errorf("failed to send the multicall: %w", err) } @@ -2860,12 +2873,12 @@ func (lane *CCIPLane) SendRequests(noOfRequests int, gasLimit *big.Int) error { gasLimit, ) if err != nil { - stat.UpdateState(lane.Logger, 0, testreporters.TX, txConfirmationDur, testreporters.Failure) + stat.UpdateState(lane.Logger, 0, testreporters.TX, txConfirmationDur, testreporters.Failure, nil) return fmt.Errorf("could not send request: %w", err) } err = lane.Source.Common.ChainClient.WaitForEvents() if err != nil { - stat.UpdateState(lane.Logger, 0, testreporters.TX, txConfirmationDur, testreporters.Failure) + stat.UpdateState(lane.Logger, 0, testreporters.TX, txConfirmationDur, testreporters.Failure, nil) return fmt.Errorf("could not send request: %w", err) } @@ -2875,22 +2888,11 @@ func (lane *CCIPLane) SendRequests(noOfRequests int, gasLimit *big.Int) error { noOfTokens++ } } - rcpt, err := lane.AddToSentReqs(txHash, []*testreporters.RequestStat{stat}) + _, err = lane.AddToSentReqs(txHash, []*testreporters.RequestStat{stat}) if err != nil { return err } - var gasUsed uint64 - if rcpt != nil { - gasUsed = rcpt.GasUsed - } - stat.UpdateState(lane.Logger, 0, - testreporters.TX, txConfirmationDur, testreporters.Success, testreporters.TransactionStats{ - Fee: fee.String(), - GasUsed: gasUsed, - TxHash: rcpt.TxHash.Hex(), - NoOfTokensSent: noOfTokens, - MessageBytesLength: lane.Source.MsgDataLength, - }) + stat.UpdateState(lane.Logger, 0, testreporters.TX, txConfirmationDur, testreporters.Success, nil) lane.TotalFee = bigmath.Add(lane.TotalFee, fee) } @@ -3144,7 +3146,7 @@ func (lane *CCIPLane) ValidateRequestByTxHash(txHash common.Hash, opts validatio return phaseErr } - sourceLogFinalizedAt, _, err := lane.Source.AssertSendRequestedLogFinalized(lane.Logger, txHash, ccipSendReqGenAt, reqStats) + sourceLogFinalizedAt, _, err := lane.Source.AssertSendRequestedLogFinalized(lane.Logger, txHash, msgLogs, ccipSendReqGenAt, reqStats) if shouldReturn, phaseErr := isPhaseValid(lane.Logger, testreporters.SourceLogFinalized, opts, err); shouldReturn { return phaseErr } @@ -3332,6 +3334,7 @@ func (lane *CCIPLane) StartEventWatchers() error { DataLength: len(e.Message.Data), NoOfTokens: len(e.Message.TokenAmounts), Raw: e.Raw, + Fee: e.Message.FeeTokenAmount, })) } else { lane.Source.CCIPSendRequestedWatcher.Store(e.Raw.TxHash.Hex(), []*contracts.SendReqEventData{ @@ -3341,6 +3344,7 @@ func (lane *CCIPLane) StartEventWatchers() error { DataLength: len(e.Message.Data), NoOfTokens: len(e.Message.TokenAmounts), Raw: e.Raw, + Fee: e.Message.FeeTokenAmount, }, }) } diff --git a/integration-tests/ccip-tests/contracts/contract_models.go b/integration-tests/ccip-tests/contracts/contract_models.go index 28dec6aedb..7ab92a673c 100644 --- a/integration-tests/ccip-tests/contracts/contract_models.go +++ b/integration-tests/ccip-tests/contracts/contract_models.go @@ -1497,6 +1497,7 @@ type SendReqEventData struct { DataLength int NoOfTokens int Raw types.Log + Fee *big.Int } type OnRampWrapper struct { diff --git a/integration-tests/ccip-tests/load/ccip_loadgen.go b/integration-tests/ccip-tests/load/ccip_loadgen.go index 4e142c9afe..ab74dd4509 100644 --- a/integration-tests/ccip-tests/load/ccip_loadgen.go +++ b/integration-tests/ccip-tests/load/ccip_loadgen.go @@ -11,7 +11,6 @@ import ( "time" "github.com/AlekSi/pointer" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/rs/zerolog" @@ -258,78 +257,25 @@ func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.Response { } startTime := time.Now().UTC() if feeToken != common.HexToAddress("0x0") { - sendTx, err = sourceCCIP.Common.Router.CCIPSend(destChainSelector, msg, nil) + sendTx, err = sourceCCIP.Common.Router.CCIPSendAndProcessTx(destChainSelector, msg, nil) } else { // add a bit buffer to fee - sendTx, err = sourceCCIP.Common.Router.CCIPSend(destChainSelector, msg, new(big.Int).Add(big.NewInt(1e5), fee)) + sendTx, err = sourceCCIP.Common.Router.CCIPSendAndProcessTx(destChainSelector, msg, new(big.Int).Add(big.NewInt(1e5), fee)) } if err != nil { - stats.UpdateState(&lggr, 0, testreporters.TX, time.Since(startTime), testreporters.Failure) + stats.UpdateState(&lggr, 0, testreporters.TX, time.Since(startTime), testreporters.Failure, nil) res.Error = fmt.Sprintf("ccip-send tx error %s for reqNo %d", err.Error(), msgSerialNo) res.Data = stats.StatusByPhase res.Failed = true return res } - err = sourceCCIP.Common.ChainClient.MarkTxAsSentOnL2(sendTx) - - if err != nil { - stats.UpdateState(&lggr, 0, testreporters.TX, time.Since(startTime), testreporters.Failure) - res.Error = fmt.Sprintf("reqNo %d failed to mark tx as sent on L2 %s", msgSerialNo, err.Error()) - res.Data = stats.StatusByPhase - res.Failed = true - return res - } - txConfirmationTime := time.Now().UTC() // wait for the tx to be mined, timeout is set to 10 minutes lggr.Info().Str("tx", sendTx.Hash().Hex()).Msg("waiting for tx to be mined") - ctx, cancel := context.WithTimeout(context.Background(), sourceCCIP.Common.ChainClient.GetNetworkConfig().Timeout.Duration) - defer cancel() - rcpt, err := bind.WaitMined(ctx, sourceCCIP.Common.ChainClient.DeployBackend(), sendTx) - if err != nil { - res.Error = fmt.Sprintf("ccip-send request tx not mined, err=%s", err.Error()) - res.Failed = true - res.Data = stats.StatusByPhase - return res - } - if rcpt == nil { - res.Error = "ccip-send request tx not mined, receipt is nil" - res.Failed = true - res.Data = stats.StatusByPhase - return res - } - hdr, err := c.Lane.Source.Common.ChainClient.HeaderByNumber(context.Background(), rcpt.BlockNumber) - if err == nil && hdr != nil { - txConfirmationTime = hdr.Timestamp - } lggr = lggr.With().Str("Msg Tx", sendTx.Hash().String()).Logger() - if rcpt.Status != types.ReceiptStatusSuccessful { - stats.UpdateState(&lggr, 0, testreporters.TX, txConfirmationTime.Sub(startTime), testreporters.Failure, - testreporters.TransactionStats{ - Fee: fee.String(), - GasUsed: rcpt.GasUsed, - TxHash: sendTx.Hash().Hex(), - NoOfTokensSent: len(msg.TokenAmounts), - MessageBytesLength: int64(len(msg.Data)), - }) - errReason, v, err := c.Lane.Source.Common.ChainClient.RevertReasonFromTx(rcpt.TxHash, router.RouterABI) - if err != nil { - errReason = "could not decode" - } - res.Error = fmt.Sprintf("ccip-send request receipt is not successful, errReason=%s, args =%v", errReason, v) - res.Failed = true - res.Data = stats.StatusByPhase - return res - } - stats.UpdateState(&lggr, 0, testreporters.TX, txConfirmationTime.Sub(startTime), testreporters.Success, - testreporters.TransactionStats{ - Fee: fee.String(), - GasUsed: rcpt.GasUsed, - TxHash: sendTx.Hash().Hex(), - NoOfTokensSent: len(msg.TokenAmounts), - MessageBytesLength: int64(len(msg.Data)), - }) + + stats.UpdateState(&lggr, 0, testreporters.TX, txConfirmationTime.Sub(startTime), testreporters.Success, nil) err = c.Validate(lggr, sendTx, txConfirmationTime, []*testreporters.RequestStat{stats}) if err != nil { res.Error = err.Error() @@ -356,19 +302,23 @@ func (c *CCIPE2ELoad) Validate(lggr zerolog.Logger, sendTx *types.Transaction, t if c.Lane.Source.Common.ChainClient.GetNetworkConfig().FinalityDepth == 0 && lstFinalizedBlock != 0 && lstFinalizedBlock > msgLogs[0].Raw.BlockNumber { sourceLogFinalizedAt = c.LastFinalizedTimestamp.Load() - for _, stat := range stats { + for i, stat := range stats { stat.UpdateState(&lggr, stat.SeqNum, testreporters.SourceLogFinalized, sourceLogFinalizedAt.Sub(sourceLogTime), testreporters.Success, - testreporters.TransactionStats{ - TxHash: msgLogs[0].Raw.TxHash.Hex(), - FinalizedByBlock: strconv.FormatUint(lstFinalizedBlock, 10), - FinalizedAt: sourceLogFinalizedAt.String(), + &testreporters.TransactionStats{ + TxHash: msgLogs[i].Raw.TxHash.Hex(), + FinalizedByBlock: strconv.FormatUint(lstFinalizedBlock, 10), + FinalizedAt: sourceLogFinalizedAt.String(), + Fee: msgLogs[i].Fee.String(), + NoOfTokensSent: msgLogs[i].NoOfTokens, + MessageBytesLength: int64(msgLogs[i].DataLength), + MsgID: fmt.Sprintf("0x%x", msgLogs[i].MessageId[:]), }) } } else { var finalizingBlock uint64 sourceLogFinalizedAt, finalizingBlock, err = c.Lane.Source.AssertSendRequestedLogFinalized( - &lggr, msgLogs[0].Raw.TxHash, sourceLogTime, stats) + &lggr, msgLogs[0].Raw.TxHash, msgLogs, sourceLogTime, stats) if err != nil { return err } diff --git a/integration-tests/ccip-tests/load/ccip_multicall_loadgen.go b/integration-tests/ccip-tests/load/ccip_multicall_loadgen.go index b0824ac9b1..ad3960dee2 100644 --- a/integration-tests/ccip-tests/load/ccip_multicall_loadgen.go +++ b/integration-tests/ccip-tests/load/ccip_multicall_loadgen.go @@ -1,13 +1,11 @@ package load import ( - "context" "fmt" "math/big" "testing" "time" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/prometheus/common/model" "github.com/rs/zerolog" @@ -178,33 +176,14 @@ func (m *CCIPMultiCallLoadGenerator) Call(_ *wasp.Generator) *wasp.Response { lggr := m.logger.With().Str("Msg Tx", sendTx.Hash().String()).Logger() txConfirmationTime := time.Now().UTC() - rcpt, err1 := bind.WaitMined(context.Background(), m.client.DeployBackend(), sendTx) - if err1 == nil { - hdr, err1 := m.client.HeaderByNumber(context.Background(), rcpt.BlockNumber) - if err1 == nil { - txConfirmationTime = hdr.Timestamp - } - } - var gasUsed uint64 - if rcpt != nil { - gasUsed = rcpt.GasUsed - } for _, rValues := range returnValuesByDest { if len(rValues.Stats) != len(rValues.Msgs) { res.Error = fmt.Sprintf("number of stats %d and msgs %d should be same", len(rValues.Stats), len(rValues.Msgs)) res.Failed = true return res } - for i, stat := range rValues.Stats { - msg := rValues.Msgs[i] - stat.UpdateState(&lggr, 0, testreporters.TX, startTime.Sub(txConfirmationTime), testreporters.Success, - testreporters.TransactionStats{ - Fee: msg.Fee.String(), - GasUsed: gasUsed, - TxHash: sendTx.Hash().Hex(), - NoOfTokensSent: len(msg.Msg.TokenAmounts), - MessageBytesLength: int64(len(msg.Msg.Data)), - }) + for _, stat := range rValues.Stats { + stat.UpdateState(&lggr, 0, testreporters.TX, startTime.Sub(txConfirmationTime), testreporters.Success, nil) } } diff --git a/integration-tests/ccip-tests/testconfig/README.md b/integration-tests/ccip-tests/testconfig/README.md index 6da17d1125..a402c218d0 100644 --- a/integration-tests/ccip-tests/testconfig/README.md +++ b/integration-tests/ccip-tests/testconfig/README.md @@ -528,7 +528,7 @@ Specifies the number of routers to be set up for each network. ### CCIP.Groups.[testgroup].MaxNoOfLanes Specifies the maximum number of lanes to be set up between networks. If this values is not set, the test will set up lanes between all possible pairs of networks mentioned in `selected_networks` in [CCIP.Env.Networks](#ccipenvnetworksselectednetworks). -For example, if `selected_networks = ['SIMULATED_1', 'SIMULATED_2', 'SIMULATED_3']`, and `MaxNoOfLanes` is set to 3, it denotes that the test will randomly select 3 lanes between all possible pairs `SIMULATED_1`, `SIMULATED_2`, and `SIMULATED_3` for the test run. +For example, if `selected_networks = ['SIMULATED_1', 'SIMULATED_2', 'SIMULATED_3']`, and `MaxNoOfLanes` is set to 2, it denotes that the test will select the first 2 lanes between all possible pairs `SIMULATED_1`, `SIMULATED_2`, and `SIMULATED_3` for the test run. ### CCIP.Groups.[testgroup].ChaosDuration Specifies the duration for which the chaos experiment is to be run. This is only valid if the test type is 'chaos'. diff --git a/integration-tests/ccip-tests/testconfig/tomls/ccip1.4-stress-2024-06-27/tier-a.toml b/integration-tests/ccip-tests/testconfig/tomls/ccip1.4-stress-2024-06-27/tier-a.toml index e69de29bb2..b7595906e6 100644 --- a/integration-tests/ccip-tests/testconfig/tomls/ccip1.4-stress-2024-06-27/tier-a.toml +++ b/integration-tests/ccip-tests/testconfig/tomls/ccip1.4-stress-2024-06-27/tier-a.toml @@ -0,0 +1,226 @@ +## Baseline performance test on simulated environment (with chaos) +## 40 chains / 400 lanes +## historyDepth 200 / finalityDepth 200 +## block_time = 1s +## throughput 1msg / 5s +## 20% Token, 60% DataWithToken, 15% Regular size msgs, 5% Large msgs +## +## make test_load_ccip testimage=795953128386.dkr.ecr.us-west-2.amazonaws.com/chainlink-ccip-tests:ccip-develop \ +## testname=TestLoadCCIPStableRequestTriggeringWithNetworkChaos \ +## override_toml=./testconfig/tomls/baseline.toml \ +## secret_toml=./testconfig/tomls/secrets.toml + +[CCIP] +[CCIP.ContractVersions] +PriceRegistry = '1.2.0' +OffRamp = '1.2.0' +OnRamp = '1.2.0' +TokenPool = '1.4.0' +CommitStore = '1.2.0' + +[CCIP.Env] +TTL = '8h' + +[CCIP.Env.Network] +selected_networks= ['PRIVATE-CHAIN-1', 'SLOW-CHAIN-1', 'SLOW-CHAIN-2', 'SLOW-CHAIN-3'] + +[CCIP.Env.Network.EVMNetworks.PRIVATE-CHAIN-1] +evm_name = 'private-chain-1' +evm_chain_id = 2337 +evm_urls = ['wss://ignore-this-url.com'] +evm_http_urls = ['https://ignore-this-url.com'] +evm_keys = ['59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d'] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '3m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 1 + +[CCIP.Env.Network.EVMNetworks.SLOW-CHAIN-1] +evm_name = 'slow-chain-1' +evm_chain_id = 90000001 +evm_urls = ['wss://ignore-this-url.com'] +evm_http_urls = ['https://ignore-this-url.com'] +evm_keys = ['ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '3m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 1 + +[CCIP.Env.Network.EVMNetworks.SLOW-CHAIN-2] +evm_name = 'slow-chain-2' +evm_chain_id = 90000002 +evm_urls = ['wss://ignore-this-url.com'] +evm_http_urls = ['https://ignore-this-url.com'] +evm_keys = ['ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '3m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 1 + +[CCIP.Env.Network.EVMNetworks.SLOW-CHAIN-3] +evm_name = 'slow-chain-3' +evm_chain_id = 1337 +evm_urls = ['wss://ignore-this-url.com'] +evm_http_urls = ['https://ignore-this-url.com'] +evm_keys = ['ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '3m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 1 + +[CCIP.Env.Network.AnvilConfigs.PRIVATE-CHAIN-1] +block_time = 1 +# +[CCIP.Env.Network.AnvilConfigs.SLOW-CHAIN-1] +block_time = 12 + +[CCIP.Env.Network.AnvilConfigs.SLOW-CHAIN-2] +block_time = 12 + +[CCIP.Env.Network.AnvilConfigs.SLOW-CHAIN-3] +block_time = 12 + +[CCIP.Env.NewCLCluster] +NoOfNodes = 17 +NodeMemory = '10Gi' +NodeCPU = '6' +DBMemory = '16Gi' +DBCPU = '4' +DBStorageClass = 'gp3' +PromPgExporter = true +DBCapacity = '50Gi' +IsStateful = true +DBArgs = ['shared_buffers=4096MB', 'effective_cache_size=8192MB', 'work_mem=128MB'] + +[CCIP.Env.NewCLCluster.Common] +BaseConfigTOML = """ +[Feature] +LogPoller = true +CCIP = true + +[Log] +Level = 'debug' +JSONConsole = true + +[Log.File] +MaxSize = '0b' + +[WebServer] +AllowOrigins = '*' +HTTPPort = 6688 +SecureCookies = false +HTTPWriteTimeout = '1m' + +[WebServer.RateLimit] +Authenticated = 2000 +Unauthenticated = 1000 + +[WebServer.TLS] +HTTPSPort = 0 + +[Database] +MaxIdleConns = 20 +MaxOpenConns = 30 +MigrateOnStartup = true + +[OCR2] +Enabled = true +DefaultTransactionQueueDepth = 0 + +[OCR] +Enabled = false +DefaultTransactionQueueDepth = 0 + +[P2P] +[P2P.V2] +Enabled = true +ListenAddresses = ['0.0.0.0:6690'] +AnnounceAddresses = ['0.0.0.0:6690'] +DeltaDial = '500ms' +DeltaReconcile = '5s' +""" + +#CommonChainConfigTOML = """ +#[HeadTracker] +#HistoryDepth = 200 +# +#[GasEstimator] +#PriceMax = '200 gwei' +#LimitDefault = 6000000 +#FeeCapDefault = '200 gwei' +#""" + +[CCIP.Groups] +[CCIP.Groups.load] +KeepEnvAlive = true +NoOfCommitNodes = 16 +PhaseTimeout = '40m' +NodeFunding = 1000.0 +NoOfRoutersPerPair = 2 +NoOfNetworks = 40 +MaxNoOfLanes = 400 + +[CCIP.Groups.load.OffRampConfig] +BatchGasLimit = 11000000 + +[CCIP.Groups.load.TokenConfig] +TimeoutForPriceUpdate = '15m' +NoOfTokensPerChain = 60 +NoOfTokensWithDynamicPrice = 15 +DynamicPriceUpdateInterval ='15s' +CCIPOwnerTokens = true + +[CCIP.Groups.load.LoadProfile] +TestDuration = '4h' +TimeUnit = '5s' +RequestPerUnitTime = [1] +OptimizeSpace = true +NetworkChaosDelay = '100ms' + +# to represent 20%, 60%, 15%, 5% of the total messages +[CCIP.Groups.load.LoadProfile.MsgProfile] +Frequencies = [4,12,3,1] + +[[CCIP.Groups.load.LoadProfile.MsgProfile.MsgDetails]] +MsgType = 'Token' +DestGasLimit = 0 +DataLength = 0 +NoOfTokens = 5 +AmountPerToken = 1 + +[[CCIP.Groups.load.LoadProfile.MsgProfile.MsgDetails]] +MsgType = 'DataWithToken' +DestGasLimit = 500000 +DataLength = 5000 +NoOfTokens = 5 +AmountPerToken = 1 + +[[CCIP.Groups.load.LoadProfile.MsgProfile.MsgDetails]] +MsgType = 'Data' +DestGasLimit = 800000 +DataLength = 10000 + +[[CCIP.Groups.load.LoadProfile.MsgProfile.MsgDetails]] +MsgType = 'Data' +DestGasLimit = 2500000 +DataLength = 10000 \ No newline at end of file diff --git a/integration-tests/ccip-tests/testreporters/ccip.go b/integration-tests/ccip-tests/testreporters/ccip.go index 8341735687..b567f6a629 100644 --- a/integration-tests/ccip-tests/testreporters/ccip.go +++ b/integration-tests/ccip-tests/testreporters/ccip.go @@ -57,10 +57,10 @@ type TransactionStats struct { } type PhaseStat struct { - SeqNum uint64 `json:"seq_num,omitempty"` - Duration float64 `json:"duration,omitempty"` - Status Status `json:"success"` - SendTransactionStats TransactionStats `json:"ccip_send_data,omitempty"` + SeqNum uint64 `json:"seq_num,omitempty"` + Duration float64 `json:"duration,omitempty"` + Status Status `json:"success"` + SendTransactionStats *TransactionStats `json:"ccip_send_data,omitempty"` } type RequestStat struct { @@ -77,19 +77,17 @@ func (stat *RequestStat) UpdateState( step Phase, duration time.Duration, state Status, - sendTransactionStats ...TransactionStats, + sendTransactionStats *TransactionStats, ) { durationInSec := duration.Seconds() stat.SeqNum = seqNum phaseDetails := PhaseStat{ - SeqNum: seqNum, - Duration: durationInSec, - Status: state, + SeqNum: seqNum, + Duration: durationInSec, + Status: state, + SendTransactionStats: sendTransactionStats, } - if len(sendTransactionStats) > 0 { - phaseDetails.SendTransactionStats = sendTransactionStats[0] - } - stat.StatusByPhase[step] = phaseDetails + event := lggr.Info() if seqNum != 0 { event.Uint64("seq num", seqNum) @@ -100,12 +98,17 @@ func (stat *RequestStat) UpdateState( SeqNum: seqNum, Status: state, } + stat.StatusByPhase[step] = phaseDetails lggr.Info(). Str(fmt.Sprint(E2E), string(state)). Msgf("reqNo %d", stat.ReqNo) event.Str(string(step), string(state)).Msgf("reqNo %d", stat.ReqNo) } else { event.Str(string(step), string(Success)).Msgf("reqNo %d", stat.ReqNo) + // we don't want to save phase details for TX and CCIPSendRe to avoid redundancy if these phases are successful + if step != TX && step != CCIPSendRe { + stat.StatusByPhase[step] = phaseDetails + } if step == Commit || step == ReportBlessed || step == ExecStateChanged { stat.StatusByPhase[E2E] = PhaseStat{ SeqNum: seqNum, diff --git a/integration-tests/ccip-tests/testsetups/ccip.go b/integration-tests/ccip-tests/testsetups/ccip.go index 9a892a667a..39e52164bc 100644 --- a/integration-tests/ccip-tests/testsetups/ccip.go +++ b/integration-tests/ccip-tests/testsetups/ccip.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/big" - "math/rand" "os" "regexp" "strings" @@ -223,8 +222,13 @@ func (c *CCIPTestConfig) SetNetworkPairs(lggr zerolog.Logger) error { actualNoOfNetworks := len(c.SelectedNetworks) n := c.SelectedNetworks[0] var chainIDs []int64 + existingChainIDs := make(map[uint64]struct{}) + for _, net := range c.SelectedNetworks { + existingChainIDs[uint64(net.ChainID)] = struct{}{} + } for _, id := range chainselectors.TestChainIds() { - if id == 2337 { + // if the chain id already exists in the already provided selected networks, skip it + if _, exists := existingChainIDs[id]; exists { continue } chainIDs = append(chainIDs, int64(id)) @@ -281,11 +285,8 @@ func (c *CCIPTestConfig) SetNetworkPairs(lggr zerolog.Logger) error { c.AddPairToNetworkList(c.SelectedNetworks[0], c.SelectedNetworks[1]) } - // if the number of lanes is lesser than the number of network pairs, choose a random subset of network pairs + // if the number of lanes is lesser than the number of network pairs, choose first c.TestGroupInput.MaxNoOfLanes pairs if c.TestGroupInput.MaxNoOfLanes > 0 && c.TestGroupInput.MaxNoOfLanes < len(c.NetworkPairs) { - rand.Shuffle(len(c.NetworkPairs), func(i, j int) { - c.NetworkPairs[i], c.NetworkPairs[j] = c.NetworkPairs[j], c.NetworkPairs[i] - }) c.NetworkPairs = c.NetworkPairs[:c.TestGroupInput.MaxNoOfLanes] }