From 7d2cae3660bac29ba9dd48b03d88a761c154d6ae Mon Sep 17 00:00:00 2001 From: "Abdelrahman Soliman (Boda)" <2677789+asoliman92@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:12:11 +0400 Subject: [PATCH] Integration test Setup Nodes and Add DON [CCIP-2810] (#1190) Setting up a basic test that have basic OCR3 Nodes. Each Node can access 3 EVM chains that are created and connected to each others. Check [PR](https://github.com/smartcontractkit/ccip/pull/1148) for more details on creating the chains and connecting them. As JobSpecs for CCIP are not ready. The main goal of this PR is to have the nodes and AddDon on capability registry for these nodes for each chain. Once JobSpec is ready we'll be able to test that the nodes picks up the changes and deploys the plugins, start sending, receiving,..etc. --- .../plugins/ccip_integration_tests/helpers.go | 170 +++++++++- .../{home_chain => }/home_chain_test.go | 21 +- .../integration_helpers.go | 11 - .../ccip_integration_tests/ocr3_node_test.go | 116 +++++++ .../ccip_integration_tests/ocr_node_helper.go | 294 ++++++++++++++++++ 5 files changed, 583 insertions(+), 29 deletions(-) rename core/services/ocr3/plugins/ccip_integration_tests/{home_chain => }/home_chain_test.go (83%) create mode 100644 core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go create mode 100644 core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go diff --git a/core/services/ocr3/plugins/ccip_integration_tests/helpers.go b/core/services/ocr3/plugins/ccip_integration_tests/helpers.go index 8526c62b27..18bcf6469c 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/helpers.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/helpers.go @@ -1,9 +1,13 @@ package ccip_integration_tests import ( + "bytes" "encoding/hex" "math/big" + "sort" + "strconv" "testing" + "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" @@ -11,15 +15,13 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config" - - chainsel "github.com/smartcontractkit/chain-selectors" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/arm_proxy_contract" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/evm_2_evm_multi_offramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/evm_2_evm_multi_onramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/mock_arm_contract" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/nonce_manager" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ocr3_config_encoder" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/price_registry" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/token_admin_registry" @@ -27,6 +29,12 @@ import ( kcr "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/shared/generated/link_token" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + cctypes "github.com/smartcontractkit/chainlink/v2/core/services/ccipcapability/types" + + confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" + + chainsel "github.com/smartcontractkit/chain-selectors" "github.com/stretchr/testify/require" ) @@ -38,13 +46,14 @@ var ( const ( CapabilityLabelledName = "ccip" CapabilityVersion = "v1.0.0" + NodeOperatorID = 1 ) func e18Mult(amount uint64) *big.Int { - return new(big.Int).Mul(uintBigInt(amount), uintBigInt(1e18)) + return new(big.Int).Mul(uBigInt(amount), uBigInt(1e18)) } -func uintBigInt(i uint64) *big.Int { +func uBigInt(i uint64) *big.Int { return new(big.Int).SetUint64(i) } @@ -53,7 +62,7 @@ type homeChain struct { owner *bind.TransactOpts chainID uint64 capabilityRegistry *kcr.CapabilitiesRegistry - ccipConfigContract common.Address + ccipConfig *ccip_config.CCIPConfig } type onchainUniverse struct { @@ -259,15 +268,149 @@ func setupHomeChain(t *testing.T, owner *bind.TransactOpts, backend *backends.Si require.NoError(t, err, "failed to add capabilities to the capability registry") backend.Commit() + // Add NodeOperator, for simplicity we'll add one NodeOperator only + // First NodeOperator will have NodeOperatorId = 1 + _, err = capabilityRegistry.AddNodeOperators(owner, []kcr.CapabilitiesRegistryNodeOperator{ + { + Admin: owner.From, + Name: "NodeOperator", + }, + }) + require.NoError(t, err, "failed to add node operator to the capability registry") + backend.Commit() + return homeChain{ backend: backend, owner: owner, chainID: homeChainID, capabilityRegistry: capabilityRegistry, - ccipConfigContract: capabilityConfig.Address(), + ccipConfig: capabilityConfig, } } +func sortP2PIDS(p2pIDs [][32]byte) { + sort.Slice(p2pIDs, func(i, j int) bool { + return bytes.Compare(p2pIDs[i][:], p2pIDs[j][:]) < 0 + }) +} + +func (h *homeChain) AddNodes( + t *testing.T, + p2pIDs [][32]byte, + capabilityIDs [][32]byte, +) { + // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail + sortP2PIDS(p2pIDs) + var nodeParams []kcr.CapabilitiesRegistryNodeParams + for _, p2pID := range p2pIDs { + nodeParam := kcr.CapabilitiesRegistryNodeParams{ + NodeOperatorId: NodeOperatorID, + Signer: p2pID, // Not used in tests + P2pId: p2pID, + HashedCapabilityIds: capabilityIDs, + } + nodeParams = append(nodeParams, nodeParam) + } + _, err := h.capabilityRegistry.AddNodes(h.owner, nodeParams) + require.NoError(t, err, "failed to add node operator oracles") + h.backend.Commit() +} + +func (h *homeChain) AddDON( + t *testing.T, + ccipCapabilityID [32]byte, + chainSelector uint64, + OfframpAddress []byte, + f uint8, + bootstrapP2PID [32]byte, + p2pIDs [][32]byte, + oracles []confighelper2.OracleIdentityExtra, +) { + // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail + sortP2PIDS(p2pIDs) + // First Add ChainConfig that includes all p2pIDs as readers + chainConfig := SetupConfigInfo(chainSelector, p2pIDs, FChainA, []byte(strconv.FormatUint(chainSelector, 10))) + inputConfig := []ccip_config.CCIPConfigTypesChainConfigInfo{ + chainConfig, + } + _, err := h.ccipConfig.ApplyChainConfigUpdates(h.owner, nil, inputConfig) + require.NoError(t, err) + h.backend.Commit() + + // Get OCR3 Config from helper + var schedule []int + for range oracles { + schedule = append(schedule, 1) + } + signers, transmitters, f, _, offchainConfigVersion, offchainConfig, err := ocr3confighelper.ContractSetConfigArgsForTests( + 30*time.Second, // deltaProgress + 10*time.Second, // deltaResend + 20*time.Second, // deltaInitial + 2*time.Second, // deltaRound + 20*time.Second, // deltaGrace + 10*time.Second, // deltaCertifiedCommitRequest + 10*time.Second, // deltaStage + 3, // rmax + schedule, + oracles, + []byte{}, // empty offchain config + 50*time.Millisecond, // maxDurationQuery + 5*time.Second, // maxDurationObservation + 10*time.Second, // maxDurationShouldAcceptAttestedReport + 10*time.Second, // maxDurationShouldTransmitAcceptedReport + int(f), + []byte{}) // empty OnChainConfig + require.NoError(t, err, "failed to create contract config") + + tabi, err := ocr3_config_encoder.IOCR3ConfigEncoderMetaData.GetAbi() + require.NoError(t, err) + + signersBytes := make([][]byte, len(signers)) + for i, signer := range signers { + signersBytes[i] = signer + } + + transmittersBytes := make([][]byte, len(transmitters)) + for i, transmitter := range transmitters { + // anotherErr because linting doesn't want to shadow err + parsed, anotherErr := common.ParseHexOrString(string(transmitter)) + require.NoError(t, anotherErr) + transmittersBytes[i] = parsed + } + + // Add DON on capability registry contract + var ocr3Configs []ocr3_config_encoder.CCIPConfigTypesOCR3Config + for _, pluginType := range []cctypes.PluginType{cctypes.PluginTypeCCIPCommit, cctypes.PluginTypeCCIPExec} { + ocr3Configs = append(ocr3Configs, ocr3_config_encoder.CCIPConfigTypesOCR3Config{ + PluginType: uint8(pluginType), + ChainSelector: chainSelector, + F: f, + OffchainConfigVersion: offchainConfigVersion, + OfframpAddress: OfframpAddress, + BootstrapP2PIds: [][32]byte{bootstrapP2PID}, + P2pIds: p2pIDs, + Signers: signersBytes, + Transmitters: transmittersBytes, + OffchainConfig: offchainConfig, + }) + } + + encodedCall, err := tabi.Pack("exposeOCR3Config", ocr3Configs) + require.NoError(t, err) + + // Trim first four bytes to remove function selector. + encodedConfigs := encodedCall[4:] + + _, err = h.capabilityRegistry.AddDON(h.owner, p2pIDs, []kcr.CapabilitiesRegistryCapabilityConfiguration{ + { + CapabilityId: ccipCapabilityID, + Config: encodedConfigs, + }, + }, false, false, f) + require.NoError(t, err) + h.backend.Commit() +} + func connectUniverses( t *testing.T, universes map[uint64]onchainUniverse, @@ -335,6 +478,17 @@ func setupUniverseBasics(t *testing.T, uni onchainUniverse) { uni.backend.Commit() } +func SetupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg []byte) ccip_config.CCIPConfigTypesChainConfigInfo { + return ccip_config.CCIPConfigTypesChainConfigInfo{ + ChainSelector: chainSelector, + ChainConfig: ccip_config.CCIPConfigTypesChainConfig{ + Readers: readers, + FChain: fChain, + Config: cfg, + }, + } +} + // As we can't change router contract. The contract was expecting onRamp and offRamp per lane and not per chain // In the new architecture we have only one onRamp and one offRamp per chain. // hence we add the mapping for all remote chains to the onRamp/offRamp contract of the local chain diff --git a/core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go b/core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go similarity index 83% rename from core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go rename to core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go index ab8629f8c1..ab710ff152 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go @@ -1,4 +1,4 @@ -package home_chain +package ccip_integration_tests import ( "testing" @@ -10,11 +10,12 @@ import ( libocrtypes "github.com/smartcontractkit/libocr/ragep2p/types" ccipreader "github.com/smartcontractkit/chainlink-ccip/pkg/reader" + cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3" + capcfg "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" "github.com/smartcontractkit/chainlink/v2/core/logger" - it "github.com/smartcontractkit/chainlink/v2/core/services/ocr3/plugins/ccip_integration_tests" "github.com/stretchr/testify/require" ) @@ -22,19 +23,19 @@ import ( func TestHomeChainReader(t *testing.T) { ctx := testutils.Context(t) lggr := logger.TestLogger(t) - uni := it.NewTestUniverse(ctx, t, lggr) + uni := NewTestUniverse(ctx, t, lggr) // We need 3*f + 1 p2pIDs to have enough nodes to bootstrap var arr []int64 - n := int(it.FChainA*3 + 1) + n := int(FChainA*3 + 1) for i := 0; i <= n; i++ { arr = append(arr, int64(i)) } - p2pIDs := it.P2pIDsFromInts(arr) + p2pIDs := P2pIDsFromInts(arr) uni.AddCapability(p2pIDs) //==============================Apply configs to Capability Contract================================= - chainAConf := it.SetupConfigInfo(it.ChainA, p2pIDs, it.FChainA, []byte("ChainA")) - chainBConf := it.SetupConfigInfo(it.ChainB, p2pIDs[1:], it.FChainB, []byte("ChainB")) - chainCConf := it.SetupConfigInfo(it.ChainC, p2pIDs[2:], it.FChainC, []byte("ChainC")) + chainAConf := SetupConfigInfo(ChainA, p2pIDs, FChainA, []byte("ChainA")) + chainBConf := SetupConfigInfo(ChainB, p2pIDs[1:], FChainB, []byte("ChainB")) + chainCConf := SetupConfigInfo(ChainC, p2pIDs[2:], FChainC, []byte("ChainC")) inputConfig := []capcfg.CCIPConfigTypesChainConfigInfo{ chainAConf, chainBConf, @@ -66,13 +67,13 @@ func TestHomeChainReader(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedChainConfigs, configs) //=================================Remove ChainC from OnChainConfig========================================= - _, err = uni.CcipCfg.ApplyChainConfigUpdates(uni.Transactor, []uint64{it.ChainC}, nil) + _, err = uni.CcipCfg.ApplyChainConfigUpdates(uni.Transactor, []uint64{ChainC}, nil) require.NoError(t, err) uni.Backend.Commit() time.Sleep(pollDuration * 5) // Wait for the chain reader to update configs, err = homeChain.GetAllChainConfigs() require.NoError(t, err) - delete(expectedChainConfigs, cciptypes.ChainSelector(it.ChainC)) + delete(expectedChainConfigs, cciptypes.ChainSelector(ChainC)) require.Equal(t, expectedChainConfigs, configs) //================================Close HomeChain Reader=============================== //require.NoError(t, homeChain.Close()) diff --git a/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go b/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go index a595bb2378..93942dbc18 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go @@ -294,14 +294,3 @@ func (t *TestUniverse) AddDONToRegistry( require.NoError(t.TestingT, err) t.Backend.Commit() } - -func SetupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg []byte) ccip_config.CCIPConfigTypesChainConfigInfo { - return ccip_config.CCIPConfigTypesChainConfigInfo{ - ChainSelector: chainSelector, - ChainConfig: ccip_config.CCIPConfigTypesChainConfig{ - Readers: readers, - FChain: fChain, - Config: cfg, - }, - } -} diff --git a/core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go b/core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go new file mode 100644 index 0000000000..bd5c12903d --- /dev/null +++ b/core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go @@ -0,0 +1,116 @@ +package ccip_integration_tests + +import ( + "fmt" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/hashicorp/consul/sdk/freeport" + + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey" + + "github.com/smartcontractkit/libocr/commontypes" + confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + "github.com/stretchr/testify/require" +) + +func TestIntegration_OCR3Nodes(t *testing.T) { + numChains := 3 + homeChainUni, universes := createUniverses(t, numChains) + numNodes := 4 + t.Log("creating ocr3 nodes") + var ( + oracles = make(map[uint64][]confighelper2.OracleIdentityExtra) + transmitters = make(map[uint64][]common.Address) + apps []chainlink.Application + nodes []*ocr3Node + p2pIDs [][32]byte + + // The bootstrap node will be the first node (index 0) + bootstrapPort int + bootstrapP2PID p2pkey.PeerID + bootstrappers []commontypes.BootstrapperLocator + ) + + ports := freeport.GetN(t, numNodes) + capabilitiesPorts := freeport.GetN(t, numNodes) + for i := 0; i < numNodes; i++ { + node := setupNodeOCR3(t, ports[i], capabilitiesPorts[i], bootstrappers, universes, homeChainUni) + + apps = append(apps, node.app) + for chainID, transmitter := range node.transmitters { + transmitters[chainID] = append(transmitters[chainID], transmitter) + identity := confighelper2.OracleIdentityExtra{ + OracleIdentity: confighelper2.OracleIdentity{ + OnchainPublicKey: node.keybundle.PublicKey(), + TransmitAccount: ocrtypes.Account(transmitter.Hex()), + OffchainPublicKey: node.keybundle.OffchainPublicKey(), + PeerID: node.peerID, + }, + ConfigEncryptionPublicKey: node.keybundle.ConfigEncryptionPublicKey(), + } + oracles[chainID] = append(oracles[chainID], identity) + } + nodes = append(nodes, node) + peerID, err := p2pkey.MakePeerID(node.peerID) + require.NoError(t, err) + p2pIDs = append(p2pIDs, peerID) + + // First Node is the bootstrap node + if i == 0 { + bootstrapPort = ports[i] + bootstrapP2PID = peerID + bootstrappers = []commontypes.BootstrapperLocator{ + {PeerID: node.peerID, Addrs: []string{ + fmt.Sprintf("127.0.0.1:%d", bootstrapPort), + }}, + } + } + } + + // Start committing periodically in the background for all the chains + tick := time.NewTicker(100 * time.Millisecond) + defer tick.Stop() + commitBlocksBackground(t, universes, tick) + + ctx := testutils.Context(t) + t.Log("creating ocr3 jobs") + for i := 0; i < len(nodes); i++ { + err := nodes[i].app.Start(ctx) + require.NoError(t, err) + tApp := apps[i] + t.Cleanup(func() { + require.NoError(t, tApp.Stop()) + }) + //TODO: Create Jobs and add them to the app + } + + ccipCapabilityID, err := homeChainUni.capabilityRegistry.GetHashedCapabilityId(&bind.CallOpts{ + Context: ctx, + }, CapabilityLabelledName, CapabilityVersion) + require.NoError(t, err, "failed to get hashed capability id for ccip") + require.NotEqual(t, [32]byte{}, ccipCapabilityID, "ccip capability id is empty") + + // Need to Add nodes and assign capabilities to them before creating DONS + homeChainUni.AddNodes(t, p2pIDs, [][32]byte{ccipCapabilityID}) + // Create a DON for each chain + for _, uni := range universes { + // Add nodes and give them the capability + t.Log("AddingDON for universe: ", uni.chainID) + homeChainUni.AddDON(t, + ccipCapabilityID, + uni.chainID, + uni.offramp.Address().Bytes(), + 1, // f + bootstrapP2PID, + p2pIDs, + oracles[uni.chainID], + ) + } +} diff --git a/core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go b/core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go new file mode 100644 index 0000000000..0163388183 --- /dev/null +++ b/core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go @@ -0,0 +1,294 @@ +package ccip_integration_tests + +import ( + "context" + "fmt" + "math/big" + "net/http" + "strconv" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" + gethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/jmoiron/sqlx" + "go.uber.org/zap/zapcore" + + "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" + "github.com/smartcontractkit/chainlink/v2/core/services/relay" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + v2toml "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" + evmutils "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" + "github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm" + "github.com/smartcontractkit/chainlink/v2/core/internal/cltest/heavyweight" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/logger/audit" + "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/ocr2key" + "github.com/smartcontractkit/chainlink/v2/core/utils" + "github.com/smartcontractkit/chainlink/v2/plugins" + + "github.com/smartcontractkit/libocr/commontypes" + "github.com/stretchr/testify/require" +) + +type ocr3Node struct { + app chainlink.Application + peerID string + transmitters map[uint64]common.Address + keybundle ocr2key.KeyBundle + db *sqlx.DB +} + +// setupNodeOCR3 creates a chainlink node and any associated keys in order to run +// ccip. +func setupNodeOCR3( + t *testing.T, + port int, + capabilitiesPort int, + p2pV2Bootstrappers []commontypes.BootstrapperLocator, + universes map[uint64]onchainUniverse, + homeChainUniverse homeChain, +) *ocr3Node { + // Do not want to load fixtures as they contain a dummy chainID. + cfg, db := heavyweight.FullTestDBNoFixturesV2(t, func(c *chainlink.Config, s *chainlink.Secrets) { + c.Insecure.OCRDevelopmentMode = ptr(true) // Disables ocr spec validation so we can have fast polling for the test. + + c.Feature.LogPoller = ptr(true) + + // P2P V2 configs. + c.P2P.V2.Enabled = ptr(true) + c.P2P.V2.DeltaDial = config.MustNewDuration(500 * time.Millisecond) + c.P2P.V2.DeltaReconcile = config.MustNewDuration(5 * time.Second) + c.P2P.V2.ListenAddresses = &[]string{fmt.Sprintf("127.0.0.1:%d", port)} + if len(p2pV2Bootstrappers) > 0 { + c.P2P.V2.DefaultBootstrappers = &p2pV2Bootstrappers + } + + // Enable Capabilities, This is a pre-requisite for registrySyncer to work. + // Same values as P2P.V2 except for the listen address. + c.Capabilities.Peering.V2 = c.P2P.V2 + c.Capabilities.Peering.V2.ListenAddresses = &[]string{fmt.Sprintf("127.0.0.1:%d", capabilitiesPort)} + c.Capabilities.ExternalRegistry.NetworkID = ptr(relay.NetworkEVM) + c.Capabilities.ExternalRegistry.ChainID = ptr(strconv.FormatUint(homeChainUniverse.chainID, 10)) + c.Capabilities.ExternalRegistry.Address = ptr(homeChainUniverse.capabilityRegistry.Address().String()) + + // OCR configs + c.OCR.Enabled = ptr(false) + c.OCR.DefaultTransactionQueueDepth = ptr(uint32(200)) + c.OCR2.Enabled = ptr(true) + c.OCR2.ContractPollInterval = config.MustNewDuration(5 * time.Second) + + var chains v2toml.EVMConfigs + for chainID := range universes { + chains = append(chains, createConfigV2Chain(uBigInt(chainID))) + } + c.EVM = chains + }) + + lggr := logger.TestLogger(t) + lggr.SetLogLevel(zapcore.InfoLevel) + ctx := testutils.Context(t) + clients := make(map[uint64]client.Client) + + for chainID, uni := range universes { + clients[chainID] = client.NewSimulatedBackendClient(t, uni.backend, uBigInt(chainID)) + } + + master := keystore.New(db, utils.FastScryptParams, lggr) + + kStore := KeystoreSim{ + eks: &EthKeystoreSim{ + Eth: master.Eth(), + t: t, + }, + csa: master.CSA(), + } + mailMon := mailbox.NewMonitor("ccip", lggr.Named("mailbox")) + evmOpts := chainlink.EVMFactoryConfig{ + ChainOpts: legacyevm.ChainOpts{ + AppConfig: cfg, + GenEthClient: func(i *big.Int) client.Client { + t.Log("genning eth client for chain id:", i.String()) + client, ok := clients[i.Uint64()] + if !ok { + t.Fatal("no backend for chainID", i) + } + return client + }, + MailMon: mailMon, + DS: db, + }, + CSAETHKeystore: kStore, + } + relayerFactory := chainlink.RelayerFactory{ + Logger: lggr, + LoopRegistry: plugins.NewLoopRegistry(lggr.Named("LoopRegistry"), cfg.Tracing()), + GRPCOpts: loop.GRPCOpts{}, + } + initOps := []chainlink.CoreRelayerChainInitFunc{chainlink.InitEVM(testutils.Context(t), relayerFactory, evmOpts)} + rci, err := chainlink.NewCoreRelayerChainInteroperators(initOps...) + require.NoError(t, err) + + app, err := chainlink.NewApplication(chainlink.ApplicationOpts{ + Config: cfg, + DS: db, + KeyStore: master, + RelayerChainInteroperators: rci, + Logger: lggr, + ExternalInitiatorManager: nil, + CloseLogger: lggr.Sync, + UnrestrictedHTTPClient: &http.Client{}, + RestrictedHTTPClient: &http.Client{}, + AuditLogger: audit.NoopLogger, + MailMon: mailMon, + LoopRegistry: plugins.NewLoopRegistry(lggr, cfg.Tracing()), + }) + require.NoError(t, err) + require.NoError(t, app.GetKeyStore().Unlock(ctx, "password")) + _, err = app.GetKeyStore().P2P().Create(ctx) + require.NoError(t, err) + + p2pIDs, err := app.GetKeyStore().P2P().GetAll() + require.NoError(t, err) + require.Len(t, p2pIDs, 1) + peerID := p2pIDs[0].PeerID() + // create a transmitter for each chain + transmitters := make(map[uint64]common.Address) + for chainID, uni := range universes { + backend := uni.backend + owner := uni.owner + cID := uBigInt(chainID) + addrs, err2 := app.GetKeyStore().Eth().EnabledAddressesForChain(testutils.Context(t), cID) + require.NoError(t, err2) + if len(addrs) == 1 { + // just fund the address + fundAddress(t, owner, addrs[0], assets.Ether(10).ToInt(), backend) + transmitters[chainID] = addrs[0] + } else { + // create key and fund it + _, err3 := app.GetKeyStore().Eth().Create(testutils.Context(t), cID) + require.NoError(t, err3, "failed to create key for chain", chainID) + sendingKeys, err3 := app.GetKeyStore().Eth().EnabledAddressesForChain(testutils.Context(t), cID) + require.NoError(t, err3) + require.Len(t, sendingKeys, 1) + fundAddress(t, owner, sendingKeys[0], assets.Ether(10).ToInt(), backend) + transmitters[chainID] = sendingKeys[0] + } + } + require.Len(t, transmitters, len(universes)) + + keybundle, err := app.GetKeyStore().OCR2().Create(ctx, chaintype.EVM) + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + + return &ocr3Node{ + // can't use this app because it doesn't have the right toml config + // missing bootstrapp + app: app, + peerID: peerID.Raw(), + transmitters: transmitters, + keybundle: keybundle, + db: db, + } +} + +func ptr[T any](v T) *T { return &v } + +var _ keystore.Eth = &EthKeystoreSim{} + +type EthKeystoreSim struct { + keystore.Eth + t *testing.T +} + +// override +func (e *EthKeystoreSim) SignTx(ctx context.Context, address common.Address, tx *gethtypes.Transaction, chainID *big.Int) (*gethtypes.Transaction, error) { + // always sign with chain id 1337 for the simulated backend + e.t.Log("always signing tx for chain id:", chainID.String(), "with chain id 1337, tx hash:", tx.Hash()) + return e.Eth.SignTx(ctx, address, tx, big.NewInt(1337)) +} + +type KeystoreSim struct { + eks keystore.Eth + csa keystore.CSA +} + +func (e KeystoreSim) Eth() keystore.Eth { + return e.eks +} + +func (e KeystoreSim) CSA() keystore.CSA { + return e.csa +} + +func fundAddress(t *testing.T, from *bind.TransactOpts, to common.Address, amount *big.Int, backend *backends.SimulatedBackend) { + nonce, err := backend.PendingNonceAt(testutils.Context(t), from.From) + require.NoError(t, err) + gp, err := backend.SuggestGasPrice(testutils.Context(t)) + require.NoError(t, err) + rawTx := gethtypes.NewTx(&gethtypes.LegacyTx{ + Nonce: nonce, + GasPrice: gp, + Gas: 21000, + To: &to, + Value: amount, + }) + signedTx, err := from.Signer(from.From, rawTx) + require.NoError(t, err) + err = backend.SendTransaction(testutils.Context(t), signedTx) + require.NoError(t, err) + backend.Commit() +} + +func createConfigV2Chain(chainID *big.Int) *v2toml.EVMConfig { + chain := v2toml.Defaults((*evmutils.Big)(chainID)) + chain.GasEstimator.LimitDefault = ptr(uint64(5e6)) + chain.LogPollInterval = config.MustNewDuration(100 * time.Millisecond) + chain.Transactions.ForwardersEnabled = ptr(false) + chain.FinalityDepth = ptr(uint32(2)) + return &v2toml.EVMConfig{ + ChainID: (*evmutils.Big)(chainID), + Enabled: ptr(true), + Chain: chain, + Nodes: v2toml.EVMNodes{&v2toml.Node{}}, + } +} + +// Commit blocks periodically in the background for all chains +func commitBlocksBackground(t *testing.T, universes map[uint64]onchainUniverse, tick *time.Ticker) { + t.Log("starting ticker to commit blocks") + tickCtx, tickCancel := context.WithCancel(testutils.Context(t)) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-tick.C: + for _, uni := range universes { + uni.backend.Commit() + } + case <-tickCtx.Done(): + return + } + } + }() + t.Cleanup(func() { + tickCancel() + wg.Wait() + }) +}