Integration test Setup Nodes and Add DON [CCIP-2810] (#1190)

Setting up a basic test that have basic OCR3 Nodes. Each Node can access
3 EVM chains that are created and connected to each others. Check
[PR](#1148) for more
details on creating the chains and connecting them.

As JobSpecs for CCIP are not ready. The main goal of this PR is to have
the nodes and AddDon on capability registry for these nodes for each
chain. Once JobSpec is ready we'll be able to test that the nodes picks
up the changes and deploys the plugins, start sending, receiving,..etc.

core/services/ocr3/plugins/ccip_integration_tests/helpers.go

@@ -1,32 +1,40 @@
 package ccip_integration_tests
 
 import (
+	"bytes"
 	"encoding/hex"
 	"math/big"
+	"sort"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"
-	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config"
-
-	chainsel "github.com/smartcontractkit/chain-selectors"
-
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/arm_proxy_contract"
+	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/evm_2_evm_multi_offramp"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/evm_2_evm_multi_onramp"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/mock_arm_contract"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/nonce_manager"
+	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ocr3_config_encoder"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/price_registry"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/token_admin_registry"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/weth9"
 	kcr "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/shared/generated/link_token"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
+	cctypes "github.com/smartcontractkit/chainlink/v2/core/services/ccipcapability/types"
+
+	confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper"
+	"github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper"
+
+	chainsel "github.com/smartcontractkit/chain-selectors"
 
 	"github.com/stretchr/testify/require"
 )
@@ -38,13 +46,14 @@ var (
 const (
 	CapabilityLabelledName = "ccip"
 	CapabilityVersion      = "v1.0.0"
+	NodeOperatorID         = 1
 )
 
 func e18Mult(amount uint64) *big.Int {
-	return new(big.Int).Mul(uintBigInt(amount), uintBigInt(1e18))
+	return new(big.Int).Mul(uBigInt(amount), uBigInt(1e18))
 }
 
-func uintBigInt(i uint64) *big.Int {
+func uBigInt(i uint64) *big.Int {
 	return new(big.Int).SetUint64(i)
 }
 
@@ -53,7 +62,7 @@ type homeChain struct {
 	owner              *bind.TransactOpts
 	chainID            uint64
 	capabilityRegistry *kcr.CapabilitiesRegistry
-	ccipConfigContract common.Address
+	ccipConfig         *ccip_config.CCIPConfig
 }
 
 type onchainUniverse struct {
@@ -259,15 +268,149 @@ func setupHomeChain(t *testing.T, owner *bind.TransactOpts, backend *backends.Si
 	require.NoError(t, err, "failed to add capabilities to the capability registry")
 	backend.Commit()
 
+	// Add NodeOperator, for simplicity we'll add one NodeOperator only
+	// First NodeOperator will have NodeOperatorId = 1
+	_, err = capabilityRegistry.AddNodeOperators(owner, []kcr.CapabilitiesRegistryNodeOperator{
+		{
+			Admin: owner.From,
+			Name:  "NodeOperator",
+		},
+	})
+	require.NoError(t, err, "failed to add node operator to the capability registry")
+	backend.Commit()
+
 	return homeChain{
 		backend:            backend,
 		owner:              owner,
 		chainID:            homeChainID,
 		capabilityRegistry: capabilityRegistry,
-		ccipConfigContract: capabilityConfig.Address(),
+		ccipConfig:         capabilityConfig,
 	}
 }
 
+func sortP2PIDS(p2pIDs [][32]byte) {
+	sort.Slice(p2pIDs, func(i, j int) bool {
+		return bytes.Compare(p2pIDs[i][:], p2pIDs[j][:]) < 0
+	})
+}
+
+func (h *homeChain) AddNodes(
+	t *testing.T,
+	p2pIDs [][32]byte,
+	capabilityIDs [][32]byte,
+) {
+	// Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail
+	sortP2PIDS(p2pIDs)
+	var nodeParams []kcr.CapabilitiesRegistryNodeParams
+	for _, p2pID := range p2pIDs {
+		nodeParam := kcr.CapabilitiesRegistryNodeParams{
+			NodeOperatorId:      NodeOperatorID,
+			Signer:              p2pID, // Not used in tests
+			P2pId:               p2pID,
+			HashedCapabilityIds: capabilityIDs,
+		}
+		nodeParams = append(nodeParams, nodeParam)
+	}
+	_, err := h.capabilityRegistry.AddNodes(h.owner, nodeParams)
+	require.NoError(t, err, "failed to add node operator oracles")
+	h.backend.Commit()
+}
+
+func (h *homeChain) AddDON(
+	t *testing.T,
+	ccipCapabilityID [32]byte,
+	chainSelector uint64,
+	OfframpAddress []byte,
+	f uint8,
+	bootstrapP2PID [32]byte,
+	p2pIDs [][32]byte,
+	oracles []confighelper2.OracleIdentityExtra,
+) {
+	// Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail
+	sortP2PIDS(p2pIDs)
+	// First Add ChainConfig that includes all p2pIDs as readers
+	chainConfig := SetupConfigInfo(chainSelector, p2pIDs, FChainA, []byte(strconv.FormatUint(chainSelector, 10)))
+	inputConfig := []ccip_config.CCIPConfigTypesChainConfigInfo{
+		chainConfig,
+	}
+	_, err := h.ccipConfig.ApplyChainConfigUpdates(h.owner, nil, inputConfig)
+	require.NoError(t, err)
+	h.backend.Commit()
+
+	// Get OCR3 Config from helper
+	var schedule []int
+	for range oracles {
+		schedule = append(schedule, 1)
+	}
+	signers, transmitters, f, _, offchainConfigVersion, offchainConfig, err := ocr3confighelper.ContractSetConfigArgsForTests(
+		30*time.Second, // deltaProgress
+		10*time.Second, // deltaResend
+		20*time.Second, // deltaInitial
+		2*time.Second,  // deltaRound
+		20*time.Second, // deltaGrace
+		10*time.Second, // deltaCertifiedCommitRequest
+		10*time.Second, // deltaStage
+		3,              // rmax
+		schedule,
+		oracles,
+		[]byte{},            // empty offchain config
+		50*time.Millisecond, // maxDurationQuery
+		5*time.Second,       // maxDurationObservation
+		10*time.Second,      // maxDurationShouldAcceptAttestedReport
+		10*time.Second,      // maxDurationShouldTransmitAcceptedReport
+		int(f),
+		[]byte{}) // empty OnChainConfig
+	require.NoError(t, err, "failed to create contract config")
+
+	tabi, err := ocr3_config_encoder.IOCR3ConfigEncoderMetaData.GetAbi()
+	require.NoError(t, err)
+
+	signersBytes := make([][]byte, len(signers))
+	for i, signer := range signers {
+		signersBytes[i] = signer
+	}
+
+	transmittersBytes := make([][]byte, len(transmitters))
+	for i, transmitter := range transmitters {
+		// anotherErr because linting doesn't want to shadow err
+		parsed, anotherErr := common.ParseHexOrString(string(transmitter))
+		require.NoError(t, anotherErr)
+		transmittersBytes[i] = parsed
+	}
+
+	// Add DON on capability registry contract
+	var ocr3Configs []ocr3_config_encoder.CCIPConfigTypesOCR3Config
+	for _, pluginType := range []cctypes.PluginType{cctypes.PluginTypeCCIPCommit, cctypes.PluginTypeCCIPExec} {
+		ocr3Configs = append(ocr3Configs, ocr3_config_encoder.CCIPConfigTypesOCR3Config{
+			PluginType:            uint8(pluginType),
+			ChainSelector:         chainSelector,
+			F:                     f,
+			OffchainConfigVersion: offchainConfigVersion,
+			OfframpAddress:        OfframpAddress,
+			BootstrapP2PIds:       [][32]byte{bootstrapP2PID},
+			P2pIds:                p2pIDs,
+			Signers:               signersBytes,
+			Transmitters:          transmittersBytes,
+			OffchainConfig:        offchainConfig,
+		})
+	}
+
+	encodedCall, err := tabi.Pack("exposeOCR3Config", ocr3Configs)
+	require.NoError(t, err)
+
+	// Trim first four bytes to remove function selector.
+	encodedConfigs := encodedCall[4:]
+
+	_, err = h.capabilityRegistry.AddDON(h.owner, p2pIDs, []kcr.CapabilitiesRegistryCapabilityConfiguration{
+		{
+			CapabilityId: ccipCapabilityID,
+			Config:       encodedConfigs,
+		},
+	}, false, false, f)
+	require.NoError(t, err)
+	h.backend.Commit()
+}
+
 func connectUniverses(
 	t *testing.T,
 	universes map[uint64]onchainUniverse,
@@ -335,6 +478,17 @@ func setupUniverseBasics(t *testing.T, uni onchainUniverse) {
 	uni.backend.Commit()
 }
 
+func SetupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg []byte) ccip_config.CCIPConfigTypesChainConfigInfo {
+	return ccip_config.CCIPConfigTypesChainConfigInfo{
+		ChainSelector: chainSelector,
+		ChainConfig: ccip_config.CCIPConfigTypesChainConfig{
+			Readers: readers,
+			FChain:  fChain,
+			Config:  cfg,
+		},
+	}
+}
+
 // As we can't change router contract. The contract was expecting onRamp and offRamp per lane and not per chain
 // In the new architecture we have only one onRamp and one offRamp per chain.
 // hence we add the mapping for all remote chains to the onRamp/offRamp contract of the local chain

core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go → core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go

@@ -1,4 +1,4 @@
-package home_chain
+package ccip_integration_tests
 
 import (
 	"testing"
@@ -10,31 +10,32 @@ import (
 	libocrtypes "github.com/smartcontractkit/libocr/ragep2p/types"
 
 	ccipreader "github.com/smartcontractkit/chainlink-ccip/pkg/reader"
+
 	cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3"
+
 	capcfg "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
-	it "github.com/smartcontractkit/chainlink/v2/core/services/ocr3/plugins/ccip_integration_tests"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestHomeChainReader(t *testing.T) {
 	ctx := testutils.Context(t)
 	lggr := logger.TestLogger(t)
-	uni := it.NewTestUniverse(ctx, t, lggr)
+	uni := NewTestUniverse(ctx, t, lggr)
 	// We need 3*f + 1 p2pIDs to have enough nodes to bootstrap
 	var arr []int64
-	n := int(it.FChainA*3 + 1)
+	n := int(FChainA*3 + 1)
 	for i := 0; i <= n; i++ {
 		arr = append(arr, int64(i))
 	}
-	p2pIDs := it.P2pIDsFromInts(arr)
+	p2pIDs := P2pIDsFromInts(arr)
 	uni.AddCapability(p2pIDs)
 	//==============================Apply configs to Capability Contract=================================
-	chainAConf := it.SetupConfigInfo(it.ChainA, p2pIDs, it.FChainA, []byte("ChainA"))
-	chainBConf := it.SetupConfigInfo(it.ChainB, p2pIDs[1:], it.FChainB, []byte("ChainB"))
-	chainCConf := it.SetupConfigInfo(it.ChainC, p2pIDs[2:], it.FChainC, []byte("ChainC"))
+	chainAConf := SetupConfigInfo(ChainA, p2pIDs, FChainA, []byte("ChainA"))
+	chainBConf := SetupConfigInfo(ChainB, p2pIDs[1:], FChainB, []byte("ChainB"))
+	chainCConf := SetupConfigInfo(ChainC, p2pIDs[2:], FChainC, []byte("ChainC"))
 	inputConfig := []capcfg.CCIPConfigTypesChainConfigInfo{
 		chainAConf,
 		chainBConf,
@@ -66,13 +67,13 @@ func TestHomeChainReader(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, expectedChainConfigs, configs)
 	//=================================Remove ChainC from OnChainConfig=========================================
-	_, err = uni.CcipCfg.ApplyChainConfigUpdates(uni.Transactor, []uint64{it.ChainC}, nil)
+	_, err = uni.CcipCfg.ApplyChainConfigUpdates(uni.Transactor, []uint64{ChainC}, nil)
 	require.NoError(t, err)
 	uni.Backend.Commit()
 	time.Sleep(pollDuration * 5) // Wait for the chain reader to update
 	configs, err = homeChain.GetAllChainConfigs()
 	require.NoError(t, err)
-	delete(expectedChainConfigs, cciptypes.ChainSelector(it.ChainC))
+	delete(expectedChainConfigs, cciptypes.ChainSelector(ChainC))
 	require.Equal(t, expectedChainConfigs, configs)
 	//================================Close HomeChain Reader===============================
 	//require.NoError(t, homeChain.Close())

core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go

@@ -294,14 +294,3 @@ func (t *TestUniverse) AddDONToRegistry(
 	require.NoError(t.TestingT, err)
 	t.Backend.Commit()
 }
-
-func SetupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg []byte) ccip_config.CCIPConfigTypesChainConfigInfo {
-	return ccip_config.CCIPConfigTypesChainConfigInfo{
-		ChainSelector: chainSelector,
-		ChainConfig: ccip_config.CCIPConfigTypesChainConfig{
-			Readers: readers,
-			FChain:  fChain,
-			Config:  cfg,
-		},
-	}
-}