Active / Candidate test (#14838)

* add mcms batch accumulator for running integ tests * add functions for proposing and promoting candidates in MCMS * add some more execution for mcms, breaking at timelock reverting on underlying transaction * format calls * more fixes * fixes * clean up * fix lint * review comments * review comments * fix lint * review comments * fix tests * more review comments * add sens requests * review comments * send a dummy req * add initial test for bluegreen migration * add ability to update number of nodes * working test, remove a node from candidate configuration, see working transmit * add active/candidate test without offchain confirmation * remove integration test binary * move code around * check for 0x0000 instead of nil from post-promotion candidate config * partial bluegreen * refactor to make plugin types generic * deploy job specs before transmitting message, pass tests * fix golang lint ci issues * passing changeset test * working test, no mcms * wip * passing test with candidate in mcms * passing test, skipping send txn after setting candidate * fix bug in transition code for active candidate * update changes that were made to deployment and launcher.go * get test passing, commit * update for linting in launcher.go * clean up, move transferownership into ownership file * lint * fixes from comments in the pr * go mod tidy * fix address type * moving imports around * fix issues from merging * fix goimport * fix go imports --------- Co-authored-by: AnieeG <anindita.ghosh@smartcontract.com>
smartcontractkit · Oct 24, 2024 · 600bc33 · 600bc33
1 parent 9bcacfc
commit 600bc33
Show file tree

Hide file tree

Showing 14 changed files with 461 additions and 189 deletions.
diff --git a/.gitignore b/.gitignore
@@ -75,6 +75,7 @@ __debug_bin*
 db_dumps/
 .run.id
 integration-tests/**/traces/
+integration-tests/**/integration-tests
 benchmark_report.csv
 benchmark_summary.json
 secrets.toml

diff --git a/core/capabilities/ccip/launcher/deployment.go b/core/capabilities/ccip/launcher/deployment.go
@@ -31,18 +31,29 @@ type ccipDeployment struct {
 
 // Close shuts down all OCR instances in the deployment.
 func (c *ccipDeployment) Close() error {
+	// we potentially run into this situation when
+	// trying to close an active instance that doesn't exist
+	// this check protects us from nil pointer exception
+
+	if c == nil {
+		return nil
+	}
 	var err error
 
 	// shutdown active commit instance.
-	err = multierr.Append(err, c.commit.active.Close())
+	if c.commit.active != nil {
+		err = multierr.Append(err, c.commit.active.Close())
+	}
 
 	// shutdown candidate commit instance.
 	if c.commit.candidate != nil {
 		err = multierr.Append(err, c.commit.candidate.Close())
 	}
 
 	// shutdown active exec instance.
-	err = multierr.Append(err, c.exec.active.Close())
+	if c.exec.active != nil {
+		err = multierr.Append(err, c.exec.active.Close())
+	}
 
 	// shutdown candidate exec instance.
 	if c.exec.candidate != nil {

diff --git a/core/capabilities/ccip/launcher/launcher.go b/core/capabilities/ccip/launcher/launcher.go
@@ -2,6 +2,7 @@ package launcher
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -189,6 +190,10 @@ func (l *launcher) processUpdate(updated map[registrysyncer.DonID]registrysyncer
 		if !ok {
 			return fmt.Errorf("invariant violation: expected to find CCIP DON %d in the map of running deployments", don.ID)
 		}
+		// we encounter this when a node is removed from the don
+		if prevDeployment == nil {
+			return errors.New("this node was closed")
+		}
 
 		futDeployment, err := updateDON(
 			l.lggr,
@@ -201,16 +206,19 @@ func (l *launcher) processUpdate(updated map[registrysyncer.DonID]registrysyncer
 		if err != nil {
 			return err
 		}
-		if err := futDeployment.TransitionDeployment(prevDeployment); err != nil {
-			// TODO: how to handle a failed active-candidate deployment?
-			return fmt.Errorf("failed to handle active-candidate deployment for CCIP DON %d: %w", donID, err)
-		}
+		// When we remove a node from the don, this node does not have a future deployment
+		if futDeployment != nil {
+			if err := futDeployment.TransitionDeployment(prevDeployment); err != nil {
+				// TODO: how to handle a failed active-candidate deployment?
+				return fmt.Errorf("failed to handle active-candidate deployment for CCIP DON %d: %w", donID, err)
+			}
 
-		// update state.
-		l.dons[donID] = futDeployment
-		// update the state with the latest config.
-		// this way if one of the starts errors, we don't retry all of them.
-		l.regState.IDsToDONs[donID] = updated[donID]
+			// update state.
+			l.dons[donID] = futDeployment
+			// update the state with the latest config.
+			// this way if one of the starts errors, we don't retry all of them.
+			l.regState.IDsToDONs[donID] = updated[donID]
+		}
 	}
 
 	return nil

diff --git a/integration-tests/deployment/ccip/add_chain.go b/integration-tests/deployment/ccip/add_chain.go
@@ -4,13 +4,12 @@ import (
 	"fmt"
 	"math/big"
 
-	"github.com/ethereum/go-ethereum/common"
+	"github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types"
 
 	"github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/mcms"
 	"github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment"
-	"github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/fee_quoter"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/onramp"
 )
@@ -26,7 +25,6 @@ func NewChainInboundProposal(
 ) (*timelock.MCMSWithTimelockProposal, error) {
 	// Generate proposal which enables new destination (from test router) on all source chains.
 	var batches []timelock.BatchChainOperation
-	var chains []uint64
 	for _, source := range sources {
 		enableOnRampDest, err := state.Chains[source].OnRamp.ApplyDestChainConfigUpdates(deployment.SimTransactOpts(), []onramp.OnRampDestChainConfigArgs{
 			{
@@ -64,47 +62,33 @@ func NewChainInboundProposal(
 				},
 			},
 		})
-		chains = append(chains, source)
 	}
 
 	addChainOp, err := ApplyChainConfigUpdatesOp(e, state, homeChainSel, []uint64{newChainSel})
 	if err != nil {
 		return nil, err
 	}
-	timelockAddresses, metaDataPerChain, err := BuildProposalMetadata(state, append(chains, homeChainSel))
-	if err != nil {
-		return nil, err
-	}
+
 	batches = append(batches, timelock.BatchChainOperation{
 		ChainIdentifier: mcms.ChainIdentifier(homeChainSel),
 		Batch: []mcms.Operation{
 			addChainOp,
 		},
 	})
-	return timelock.NewMCMSWithTimelockProposal(
-		"1",
-		2004259681, // TODO: should be parameterized and based on current block timestamp.
-		[]mcms.Signature{},
-		false,
-		metaDataPerChain,
-		timelockAddresses,
-		"blah", // TODO
-		batches,
-		timelock.Schedule,
-		"0s", // TODO: Should be parameterized.
-	)
+
+	return BuildProposalFromBatches(state, batches, "proposal to set new chains", 0)
 }
 
-// AddDonAndSetCandidateForCommitProposal adds new DON for destination to home chain
+// AddDonAndSetCandidateProposal adds new DON for destination to home chain
 // and sets the commit plugin config as candidateConfig for the don.
-func AddDonAndSetCandidateForCommitProposal(
+func AddDonAndSetCandidateProposal(
 	state CCIPOnChainState,
 	e deployment.Environment,
 	nodes deployment.Nodes,
 	ocrSecrets deployment.OCRSecrets,
 	homeChainSel, feedChainSel, newChainSel uint64,
 	tokenConfig TokenConfig,
-	rmnHomeAddress common.Address,
+	pluginType types.PluginType,
 ) (*timelock.MCMSWithTimelockProposal, error) {
 	newDONArgs, err := BuildOCR3ConfigForCCIPHome(
 		e.Logger,
@@ -114,7 +98,7 @@ func AddDonAndSetCandidateForCommitProposal(
 		feedChainSel,
 		tokenConfig.GetTokenInfo(e.Logger, state.Chains[newChainSel].LinkToken, state.Chains[newChainSel].Weth9),
 		nodes.NonBootstraps(),
-		rmnHomeAddress,
+		state.Chains[homeChainSel].RMNHome.Address(),
 	)
 	if err != nil {
 		return nil, err
@@ -123,36 +107,22 @@ func AddDonAndSetCandidateForCommitProposal(
 	if err != nil {
 		return nil, err
 	}
-	commitConfig, ok := newDONArgs[types.PluginTypeCCIPCommit]
+	commitConfig, ok := newDONArgs[pluginType]
 	if !ok {
 		return nil, fmt.Errorf("missing commit plugin in ocr3Configs")
 	}
 	donID := latestDon.Id + 1
-	addDonOp, err := SetCandidateCommitPluginWithAddDonOps(
+	addDonOp, err := NewDonWithCandidateOp(
 		donID, commitConfig,
 		state.Chains[homeChainSel].CapabilityRegistry,
 		nodes.NonBootstraps(),
 	)
 	if err != nil {
 		return nil, err
 	}
-	timelockAddresses, metaDataPerChain, err := BuildProposalMetadata(state, []uint64{homeChainSel})
-	if err != nil {
-		return nil, err
-	}
-	return timelock.NewMCMSWithTimelockProposal(
-		"1",
-		2004259681, // TODO: should be parameterized and based on current block timestamp.
-		[]mcms.Signature{},
-		false,
-		metaDataPerChain,
-		timelockAddresses,
-		"SetCandidate for commit And AddDon for new chain",
-		[]timelock.BatchChainOperation{{
-			ChainIdentifier: mcms.ChainIdentifier(homeChainSel),
-			Batch:           []mcms.Operation{addDonOp},
-		}},
-		timelock.Schedule,
-		"0s", // TODO: Should be parameterized.
-	)
+
+	return BuildProposalFromBatches(state, []timelock.BatchChainOperation{{
+		ChainIdentifier: mcms.ChainIdentifier(homeChainSel),
+		Batch:           []mcms.Operation{addDonOp},
+	}}, "setCandidate for commit and AddDon on new Chain", 0)
 }
diff --git a/integration-tests/deployment/ccip/add_chain_test.go b/integration-tests/deployment/ccip/add_chain_test.go
@@ -4,6 +4,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types"
+
 	"github.com/ethereum/go-ethereum/common"
 
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/rmn_home"
@@ -25,7 +27,7 @@ import (
 
 func TestAddChainInbound(t *testing.T) {
 	// 4 chains where the 4th is added after initial deployment.
-	e := NewMemoryEnvironmentWithJobs(t, logger.TestLogger(t), 4)
+	e := NewMemoryEnvironmentWithJobs(t, logger.TestLogger(t), 4, 4)
 	state, err := LoadOnchainState(e.Env, e.Ab)
 	require.NoError(t, err)
 	// Take first non-home chain as the new chain.
@@ -148,14 +150,14 @@ func TestAddChainInbound(t *testing.T) {
 	//SendRequest(t, e.Env, state, initialDeploy[0], newChain, true)
 
 	t.Logf("Executing add don and set candidate proposal for commit plugin on chain %d", newChain)
-	addDonProp, err := AddDonAndSetCandidateForCommitProposal(state, e.Env, nodes, deployment.XXXGenerateTestOCRSecrets(), e.HomeChainSel, e.FeedChainSel, newChain, tokenConfig, common.HexToAddress(rmnHomeAddress))
+	addDonProp, err := AddDonAndSetCandidateProposal(state, e.Env, nodes, deployment.XXXGenerateTestOCRSecrets(), e.HomeChainSel, e.FeedChainSel, newChain, tokenConfig, types.PluginTypeCCIPCommit)
 	require.NoError(t, err)
 
 	addDonExec := SignProposal(t, e.Env, addDonProp)
 	ExecuteProposal(t, e.Env, addDonExec, state, e.HomeChainSel)
 
 	t.Logf("Executing promote candidate proposal for exec plugin on chain %d", newChain)
-	setCandidateForExecProposal, err := SetCandidateExecPluginProposal(state, e.Env, nodes, deployment.XXXGenerateTestOCRSecrets(), e.HomeChainSel, e.FeedChainSel, newChain, tokenConfig, common.HexToAddress(rmnHomeAddress))
+	setCandidateForExecProposal, err := SetCandidatePluginProposal(state, e.Env, nodes, deployment.XXXGenerateTestOCRSecrets(), e.HomeChainSel, e.FeedChainSel, newChain, tokenConfig, types.PluginTypeCCIPExec)
 	require.NoError(t, err)
 	setCandidateForExecExec := SignProposal(t, e.Env, setCandidateForExecProposal)
 	ExecuteProposal(t, e.Env, setCandidateForExecExec, state, e.HomeChainSel)

diff --git a/integration-tests/deployment/ccip/add_lane_test.go b/integration-tests/deployment/ccip/add_lane_test.go
@@ -16,7 +16,7 @@ func TestAddLane(t *testing.T) {
 	// TODO: The offchain code doesn't yet support partial lane
 	// enablement, need to address then re-enable this test.
 	t.Skip()
-	e := NewMemoryEnvironmentWithJobs(t, logger.TestLogger(t), 3)
+	e := NewMemoryEnvironmentWithJobs(t, logger.TestLogger(t), 3, 4)
 	// Here we have CR + nodes set up, but no CCIP contracts deployed.
 	state, err := LoadOnchainState(e.Env, e.Ab)
 	require.NoError(t, err)