CCIP-3391:Port reorg test from CCIP (#14385)

* CCIP-3391:Port reorg test from CCIP

* Add reorg test to the pipeline and fix lint

* Fix lint

* fix override config path

* Debug config override

* Fix override variable name as BASE64_CONFIG_OVERRIDE

* log

* Override should happen before validating it

* clean up the logs

* Remove chainlink image

.github/e2e-tests.yml

@@ -1004,7 +1004,43 @@ runner-test-matrix:
       - Nightly E2E Tests
     test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPOffRampAggRateLimit$ -timeout 30m -count=1 -test.parallel=1 -json
     test_env_vars:
-      E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2      
+      E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
+
+  - id: ccip-tests/smoke/ccip_test.go:^TestSmokeCCIPReorgBelowFinality$
+    path: integration-tests/ccip-tests/smoke/ccip_test.go
+    test_env_type: docker
+    runs_on: ubuntu-latest
+    workflows:
+      - PR E2E CCIP Tests
+      - Nightly E2E Tests
+    test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPReorgBelowFinality$ -timeout 30m -count=1 -test.parallel=1 -json
+    test_env_vars:
+      E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
+    test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml
+
+  - id: ccip-tests/smoke/ccip_test.go:^TestSmokeCCIPReorgAboveFinalityAtDestination$
+    path: integration-tests/ccip-tests/smoke/ccip_test.go
+    test_env_type: docker
+    runs_on: ubuntu-latest
+    workflows:
+      - PR E2E CCIP Tests
+      - Nightly E2E Tests
+    test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPReorgAboveFinalityAtDestination$ -timeout 30m -count=1 -test.parallel=1 -json
+    test_env_vars:
+      E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
+    test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml
+
+  - id: ccip-tests/smoke/ccip_test.go:^TestSmokeCCIPReorgAboveFinalityAtSource$
+    path: integration-tests/ccip-tests/smoke/ccip_test.go
+    test_env_type: docker
+    runs_on: ubuntu-latest
+    workflows:
+      - PR E2E CCIP Tests
+      - Nightly E2E Tests
+    test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPReorgAboveFinalityAtSource$ -timeout 30m -count=1 -test.parallel=1 -json
+    test_env_vars:
+      E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
+    test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml
 
   - id: ccip-tests/chaos/ccip_test.go
     path: integration-tests/ccip-tests/chaos/ccip_test.go

.github/workflows/ccip-load-tests.yml

@@ -157,14 +157,14 @@ jobs:
         run: |
           # if the matrix.type.config_path is set, use it as the override config
           if [ -n "${{ matrix.type.config_path }}" ]; then
-              BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ${{ matrix.type.config_path }})
-              echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
-              echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
+              BASE64_CONFIG_OVERRIDE=$(base64 -w 0 -i ${{ matrix.type.config_path }})
+              echo ::add-mask::$BASE64_CONFIG_OVERRIDE
+              echo "base_64_override=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
           fi
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH)
-            echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
-            echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
+            BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH)
+            echo ::add-mask::$BASE64_CONFIG_OVERRIDE
+            echo "base_64_override=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
           fi
       - name: step summary
         shell: bash
@@ -190,9 +190,8 @@ jobs:
           RR_MEM: 8Gi
           RR_CPU: 4
           TEST_TRIGGERED_BY: ccip-load-test-ci-${{ matrix.type.name }}
-          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
-          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
-          E2E_TEST_LOGGING_RUN_ID: ${{ github.run_id }}       
+          BASE64_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
+          TEST_BASE64_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
           E2E_TEST_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
           E2E_TEST_CHAINLINK_VERSION: ${{ env.CHAINLINK_VERSION }}
           E2E_TEST_LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
@@ -300,4 +299,4 @@ jobs:
           slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }}
           slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }}
 
-  # End Reporting Jobs
+  # End Reporting Jobs

core/services/ocr2/plugins/ccip/clo_ccip_integration_test.go

@@ -6,6 +6,8 @@ import (
 	"math/big"
 	"testing"
 
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip"
+
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/stretchr/testify/assert"
@@ -20,7 +22,15 @@ import (
 )
 
 func Test_CLOSpecApprovalFlow_pipeline(t *testing.T) {
-	ccipTH := integrationtesthelpers.SetupCCIPIntegrationTH(t, testhelpers.SourceChainID, testhelpers.SourceChainSelector, testhelpers.DestChainID, testhelpers.DestChainSelector)
+	ccipTH := integrationtesthelpers.SetupCCIPIntegrationTH(
+		t,
+		testhelpers.SourceChainID,
+		testhelpers.SourceChainSelector,
+		testhelpers.DestChainID,
+		testhelpers.DestChainSelector,
+		ccip.DefaultSourceFinalityDepth,
+		ccip.DefaultDestFinalityDepth,
+	)
 
 	tokenPricesUSDPipeline, linkUSD, ethUSD := ccipTH.CreatePricesPipeline(t)
 	defer linkUSD.Close()
@@ -30,7 +40,15 @@ func Test_CLOSpecApprovalFlow_pipeline(t *testing.T) {
 }
 
 func Test_CLOSpecApprovalFlow_dynamicPriceGetter(t *testing.T) {
-	ccipTH := integrationtesthelpers.SetupCCIPIntegrationTH(t, testhelpers.SourceChainID, testhelpers.SourceChainSelector, testhelpers.DestChainID, testhelpers.DestChainSelector)
+	ccipTH := integrationtesthelpers.SetupCCIPIntegrationTH(
+		t,
+		testhelpers.SourceChainID,
+		testhelpers.SourceChainSelector,
+		testhelpers.DestChainID,
+		testhelpers.DestChainSelector,
+		ccip.DefaultSourceFinalityDepth,
+		ccip.DefaultDestFinalityDepth,
+	)
 
 	//Set up the aggregators here to avoid modifying ccipTH.
 	dstLinkAddr := ccipTH.Dest.LinkToken.Address()

core/services/ocr2/plugins/ccip/integration_test.go

@@ -9,6 +9,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip"
+
 	"github.com/ethereum/go-ethereum/common"
 	gethtypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/stretchr/testify/assert"
@@ -46,7 +49,15 @@ func TestIntegration_CCIP(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			ccipTH := integrationtesthelpers.SetupCCIPIntegrationTH(t, testhelpers.SourceChainID, testhelpers.SourceChainSelector, testhelpers.DestChainID, testhelpers.DestChainSelector)
+			ccipTH := integrationtesthelpers.SetupCCIPIntegrationTH(
+				t,
+				testhelpers.SourceChainID,
+				testhelpers.SourceChainSelector,
+				testhelpers.DestChainID,
+				testhelpers.DestChainSelector,
+				ccip.DefaultSourceFinalityDepth,
+				ccip.DefaultDestFinalityDepth,
+			)
 
 			tokenPricesUSDPipeline := ""
 			priceGetterConfigJson := ""
@@ -629,3 +640,71 @@ func TestIntegration_CCIP(t *testing.T) {
 		})
 	}
 }
+
+// TestReorg ensures that CCIP works even when a below finality depth reorg happens
+func TestReorg(t *testing.T) {
+	// We need higher finality depth on the destination to perform reorg deep enough to revert commit and execution reports
+	destinationFinalityDepth := uint32(50)
+	ccipTH := integrationtesthelpers.SetupCCIPIntegrationTH(
+		t,
+		testhelpers.SourceChainID,
+		testhelpers.SourceChainSelector,
+		testhelpers.DestChainID,
+		testhelpers.DestChainSelector,
+		ccip.DefaultSourceFinalityDepth,
+		destinationFinalityDepth,
+	)
+	testPricePipeline, linkUSD, ethUSD := ccipTH.CreatePricesPipeline(t)
+	defer linkUSD.Close()
+	defer ethUSD.Close()
+	ccipTH.SetUpNodesAndJobs(t, testPricePipeline, "", "")
+
+	gasLimit := big.NewInt(200_00)
+	tokenAmount := big.NewInt(1)
+
+	forkBlock, err := ccipTH.Dest.Chain.BlockByNumber(context.Background(), nil)
+	require.NoError(t, err, "Error while fetching the destination chain current block number")
+
+	// Adjust time to start next blocks with timestamps two hours after the fork block.
+	// This is critical to have two forks with different block_timestamps.
+	err = ccipTH.Dest.Chain.AdjustTime(2 * time.Hour)
+	require.NoError(t, err, "Error while adjusting the destination chain time")
+	ccipTH.Dest.Chain.Commit()
+
+	// Send request for the first time and make sure it's executed on the destination
+	ccipTH.SendMessage(t, gasLimit, tokenAmount, ccipTH.Dest.Receivers[0].Receiver.Address())
+	ccipTH.Dest.User.GasLimit = 100000
+	ccipTH.EventuallySendRequested(t, uint64(1))
+	ccipTH.EventuallyReportCommitted(t, 1)
+	executionLog := ccipTH.AllNodesHaveExecutedSeqNums(t, 1, 1)
+	ccipTH.AssertExecState(t, executionLog[0], testhelpers.ExecutionStateSuccess)
+
+	currentBlock, err := ccipTH.Dest.Chain.BlockByNumber(context.Background(), nil)
+	require.NoError(t, err, "Error while fetching the current block number of destination chain")
+
+	// Reorg back to the `forkBlock`. Next blocks in the fork will have block_timestamps right after the fork,
+	// but before the 2 hours interval defined above for the canonical chain
+	require.NoError(t, ccipTH.Dest.Chain.Fork(testutils.Context(t), forkBlock.Hash()),
+		"Error while forking the chain")
+	// Make sure that fork is longer than the canonical chain to enforce switch
+	noOfBlocks := uint(currentBlock.NumberU64() - forkBlock.NumberU64())
+	for i := uint(0); i < noOfBlocks+1; i++ {
+		ccipTH.Dest.Chain.Commit()
+	}
+
+	// State of the chain (block_timestamps) after reorg:
+	//            / --> block1 (02:01) --> block2 (02:02) --> commit report (02:03) --> ...
+	// forkBlock (00:00) --> block1' (00:01) --> block2' (00:02) --> commit report' (00:03) --> ...
+
+	// CCIP should commit and executed messages that was reorged away
+	ccipTH.EventuallyReportCommitted(t, 1)
+	executionLog = ccipTH.AllNodesHaveExecutedSeqNums(t, 1, 1)
+	ccipTH.AssertExecState(t, executionLog[0], testhelpers.ExecutionStateSuccess)
+
+	// Sending another message and make sure it's executed on the destination
+	ccipTH.SendMessage(t, gasLimit, tokenAmount, ccipTH.Dest.Receivers[0].Receiver.Address())
+	ccipTH.EventuallySendRequested(t, uint64(2))
+	ccipTH.EventuallyReportCommitted(t, 2)
+	executionLog = ccipTH.AllNodesHaveExecutedSeqNums(t, 1, 2)
+	ccipTH.AssertExecState(t, executionLog[0], testhelpers.ExecutionStateSuccess)
+}

core/services/ocr2/plugins/ccip/testhelpers/ccip_contracts.go

@@ -179,6 +179,7 @@ type Common struct {
 	ARMProxy           *rmn_proxy_contract.RMNProxyContract
 	PriceRegistry      *price_registry_1_2_0.PriceRegistry
 	TokenAdminRegistry *token_admin_registry.TokenAdminRegistry
+	FinalityDepth      uint32
 }
 
 type SourceChain struct {
@@ -652,7 +653,8 @@ func SetAdminAndRegisterPool(t *testing.T,
 	chain.Commit()
 }
 
-func SetupCCIPContracts(t *testing.T, sourceChainID, sourceChainSelector, destChainID, destChainSelector uint64) CCIPContracts {
+func SetupCCIPContracts(t *testing.T, sourceChainID, sourceChainSelector, destChainID, destChainSelector uint64,
+	sourceFinalityDepth, destFinalityDepth uint32) CCIPContracts {
 	sourceChain, sourceUser := SetupChain(t)
 	destChain, destUser := SetupChain(t)
 
@@ -1177,6 +1179,7 @@ func SetupCCIPContracts(t *testing.T, sourceChainID, sourceChainSelector, destCh
 			WrappedNative:      sourceWrapped,
 			WrappedNativePool:  sourceWeth9Pool,
 			TokenAdminRegistry: sourceTokenAdminRegistry,
+			FinalityDepth:      sourceFinalityDepth,
 		},
 		Router: sourceRouter,
 		OnRamp: onRamp,
@@ -1196,6 +1199,7 @@ func SetupCCIPContracts(t *testing.T, sourceChainID, sourceChainSelector, destCh
 			WrappedNative:      destWrapped,
 			WrappedNativePool:  destWrappedPool,
 			TokenAdminRegistry: destTokenAdminRegistry,
+			FinalityDepth:      destFinalityDepth,
 		},
 		CommitStoreHelper: commitStoreHelper,
 		CommitStore:       commitStore,

core/services/ocr2/plugins/ccip/testhelpers/integration/chainlink.go

@@ -371,6 +371,7 @@ func setupNodeCCIP(
 	sourceChainID *big.Int, destChainID *big.Int,
 	bootstrapPeerID string,
 	bootstrapPort int64,
+	sourceFinalityDepth, destFinalityDepth uint32,
 ) (chainlink.Application, string, common.Address, ocr2key.KeyBundle) {
 	trueRef, falseRef := true, false
 
@@ -404,7 +405,7 @@ func setupNodeCCIP(
 		c.P2P.V2.ListenAddresses = &p2pAddresses
 		c.P2P.V2.AnnounceAddresses = &p2pAddresses
 
-		c.EVM = []*v2.EVMConfig{createConfigV2Chain(sourceChainID), createConfigV2Chain(destChainID)}
+		c.EVM = []*v2.EVMConfig{createConfigV2Chain(sourceChainID, sourceFinalityDepth), createConfigV2Chain(destChainID, destFinalityDepth)}
 
 		if bootstrapPeerID != "" {
 			// Supply the bootstrap IP and port as a V2 peer address
@@ -526,21 +527,20 @@ func setupNodeCCIP(
 	return app, peerID.Raw(), transmitter, kb
 }
 
-func createConfigV2Chain(chainId *big.Int) *v2.EVMConfig {
+func createConfigV2Chain(chainID *big.Int, finalityDepth uint32) *v2.EVMConfig {
 	// NOTE: For the executor jobs, the default of 500k is insufficient for a 3 message batch
 	defaultGasLimit := uint64(5000000)
 	tr := true
 
-	sourceC := v2.Defaults((*evmUtils.Big)(chainId))
+	sourceC := v2.Defaults((*evmUtils.Big)(chainID))
 	sourceC.GasEstimator.LimitDefault = &defaultGasLimit
 	fixedPrice := "FixedPrice"
 	sourceC.GasEstimator.Mode = &fixedPrice
 	d, _ := config.NewDuration(100 * time.Millisecond)
 	sourceC.LogPollInterval = &d
-	fd := uint32(2)
-	sourceC.FinalityDepth = &fd
+	sourceC.FinalityDepth = &finalityDepth
 	return &v2.EVMConfig{
-		ChainID: (*evmUtils.Big)(chainId),
+		ChainID: (*evmUtils.Big)(chainID),
 		Enabled: &tr,
 		Chain:   sourceC,
 		Nodes:   v2.EVMNodes{&v2.Node{}},
@@ -553,9 +553,11 @@ type CCIPIntegrationTestHarness struct {
 	Bootstrap Node
 }
 
-func SetupCCIPIntegrationTH(t *testing.T, sourceChainID, sourceChainSelector, destChainId, destChainSelector uint64) CCIPIntegrationTestHarness {
+func SetupCCIPIntegrationTH(t *testing.T, sourceChainID, sourceChainSelector, destChainID, destChainSelector uint64,
+	sourceFinalityDepth, destFinalityDepth uint32) CCIPIntegrationTestHarness {
 	return CCIPIntegrationTestHarness{
-		CCIPContracts: testhelpers.SetupCCIPContracts(t, sourceChainID, sourceChainSelector, destChainId, destChainSelector),
+		CCIPContracts: testhelpers.SetupCCIPContracts(t, sourceChainID, sourceChainSelector, destChainID,
+			destChainSelector, sourceFinalityDepth, destFinalityDepth),
 	}
 }
 
@@ -928,7 +930,8 @@ func (c *CCIPIntegrationTestHarness) ConsistentlyReportNotCommitted(t *testing.T
 func (c *CCIPIntegrationTestHarness) SetupAndStartNodes(ctx context.Context, t *testing.T, bootstrapNodePort int64) (Node, []Node, int64) {
 	appBootstrap, bootstrapPeerID, bootstrapTransmitter, bootstrapKb := setupNodeCCIP(t, c.Dest.User, bootstrapNodePort,
 		"bootstrap_ccip", c.Source.Chain, c.Dest.Chain, big.NewInt(0).SetUint64(c.Source.ChainID),
-		big.NewInt(0).SetUint64(c.Dest.ChainID), "", 0)
+		big.NewInt(0).SetUint64(c.Dest.ChainID), "", 0, c.Source.FinalityDepth,
+		c.Dest.FinalityDepth)
 	var (
 		oracles []confighelper.OracleIdentityExtra
 		nodes   []Node
@@ -956,6 +959,8 @@ func (c *CCIPIntegrationTestHarness) SetupAndStartNodes(ctx context.Context, t *
 			big.NewInt(0).SetUint64(c.Dest.ChainID),
 			bootstrapPeerID,
 			bootstrapNodePort,
+			c.Source.FinalityDepth,
+			c.Dest.FinalityDepth,
 		)
 		nodes = append(nodes, Node{
 			App:         app,

core/services/ocr2/plugins/ccip/vars.go

@@ -5,10 +5,12 @@ import (
 )
 
 const (
-	MaxQueryLength       = 0       // empty for both plugins
-	MaxObservationLength = 250_000 // plugins's Observation should make sure to cap to this limit
-	CommitPluginLabel    = "commit"
-	ExecPluginLabel      = "exec"
+	MaxQueryLength             = 0       // empty for both plugins
+	MaxObservationLength       = 250_000 // plugins's Observation should make sure to cap to this limit
+	CommitPluginLabel          = "commit"
+	ExecPluginLabel            = "exec"
+	DefaultSourceFinalityDepth = uint32(2)
+	DefaultDestFinalityDepth   = uint32(2)
 )
 
 var ErrChainIsNotHealthy = errors.New("lane processing is stopped because of healthcheck failure, please see crit logs")

integration-tests/ccip-tests/Makefile

@@ -4,10 +4,10 @@
 set_config:
 	if [ -s "$(override_toml)" ]; then \
   			echo "Overriding config with $(override_toml)"; \
-			echo "export BASE64_CCIP_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" > ./testconfig/override/.env; \
-        	echo "export TEST_BASE64_CCIP_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" >> ./testconfig/override/.env; \
-			echo "BASE64_CCIP_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" > ./testconfig/override/debug.env; \
-			echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" >> ./testconfig/override/debug.env; \
+			echo "export BASE64_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" > ./testconfig/override/.env; \
+        	echo "export TEST_BASE64_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" >> ./testconfig/override/.env; \
+			echo "BASE64_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" > ./testconfig/override/debug.env; \
+			echo "TEST_BASE64_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" >> ./testconfig/override/debug.env; \
     else \
       		echo "No override config found, using default config"; \
         	echo  > ./testconfig/override/.env; \
@@ -56,8 +56,8 @@ test_smoke_ccip: set_config
 test_smoke_ccip_default: set_config
 	source ./testconfig/override/.env && \
 	DATABASE_URL=postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable	\
-	BASE64_CCIP_CONFIG_OVERRIDE=""	\
-	TEST_BASE64_CCIP_CONFIG_OVERRIDE=""	\
+	BASE64_CONFIG_OVERRIDE=""	\
+	TEST_BASE64_CONFIG_OVERRIDE=""	\
 	ENV_JOB_IMAGE=""  \
 	TEST_SUITE=smoke \
 	TEST_ARGS="-test.timeout 900h" \