From 20d60499de4916a86125b8010b20a13e722bd1b8 Mon Sep 17 00:00:00 2001 From: dimkouv Date: Fri, 13 Dec 2024 10:31:02 +0200 Subject: [PATCH 1/3] re-enable rmn tests with larger runners --- .github/e2e-tests.yml | 16 ++++++++-------- integration-tests/smoke/ccip/ccip_rmn_test.go | 11 ++--------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 1bf55a64418..93ddbb564b6 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -966,7 +966,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_TwoMessagesOnTwoLanesIncludingBatching$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -982,7 +982,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_MultipleMessagesOnOneLaneNoWaitForExec$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -998,7 +998,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_NotEnoughObservers$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -1014,7 +1014,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_DifferentSigners$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -1030,7 +1030,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_NotEnoughSigners$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -1046,7 +1046,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_DifferentRmnNodesForDifferentChains$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -1062,7 +1062,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_TwoMessagesOneSourceChainCursed$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -1078,7 +1078,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_GlobalCurseTwoMessagesOnTwoLanes$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests diff --git a/integration-tests/smoke/ccip/ccip_rmn_test.go b/integration-tests/smoke/ccip/ccip_rmn_test.go index c22f9bcf20e..d621d99dcc0 100644 --- a/integration-tests/smoke/ccip/ccip_rmn_test.go +++ b/integration-tests/smoke/ccip/ccip_rmn_test.go @@ -18,11 +18,12 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/chainlink-ccip/pkg/reader" "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/osutil" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" + "github.com/smartcontractkit/chainlink-ccip/pkg/reader" + "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" "github.com/smartcontractkit/chainlink/deployment/environment/devenv" @@ -35,7 +36,6 @@ import ( ) func TestRMN_TwoMessagesOnTwoLanesIncludingBatching(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "messages on two lanes including batching", waitForExec: true, @@ -59,7 +59,6 @@ func TestRMN_TwoMessagesOnTwoLanesIncludingBatching(t *testing.T) { } func TestRMN_MultipleMessagesOnOneLaneNoWaitForExec(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "multiple messages for rmn batching inspection and one rmn node down", waitForExec: false, // do not wait for execution reports @@ -82,7 +81,6 @@ func TestRMN_MultipleMessagesOnOneLaneNoWaitForExec(t *testing.T) { } func TestRMN_NotEnoughObservers(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "one message but not enough observers, should not get a commit report", passIfNoCommitAfter: 15 * time.Second, @@ -105,7 +103,6 @@ func TestRMN_NotEnoughObservers(t *testing.T) { } func TestRMN_DifferentSigners(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "different signers and different observers", homeChainConfig: homeChainConfig{ @@ -130,7 +127,6 @@ func TestRMN_DifferentSigners(t *testing.T) { } func TestRMN_NotEnoughSigners(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "different signers and different observers", passIfNoCommitAfter: 15 * time.Second, @@ -156,7 +152,6 @@ func TestRMN_NotEnoughSigners(t *testing.T) { } func TestRMN_DifferentRmnNodesForDifferentChains(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "different rmn nodes support different chains", waitForExec: false, @@ -183,7 +178,6 @@ func TestRMN_DifferentRmnNodesForDifferentChains(t *testing.T) { } func TestRMN_TwoMessagesOneSourceChainCursed(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "two messages, one source chain is cursed", passIfNoCommitAfter: 15 * time.Second, @@ -210,7 +204,6 @@ func TestRMN_TwoMessagesOneSourceChainCursed(t *testing.T) { } func TestRMN_GlobalCurseTwoMessagesOnTwoLanes(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "global curse messages on two lanes", waitForExec: false, From 50ceda8fd99821c7e2354efc60443d1a06749f10 Mon Sep 17 00:00:00 2001 From: dimkouv Date: Fri, 13 Dec 2024 11:09:16 +0200 Subject: [PATCH 2/3] add uncurse test case --- integration-tests/smoke/ccip/ccip_rmn_test.go | 80 ++++++++++++++++--- 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/integration-tests/smoke/ccip/ccip_rmn_test.go b/integration-tests/smoke/ccip/ccip_rmn_test.go index d621d99dcc0..014fe3a2288 100644 --- a/integration-tests/smoke/ccip/ccip_rmn_test.go +++ b/integration-tests/smoke/ccip/ccip_rmn_test.go @@ -179,11 +179,14 @@ func TestRMN_DifferentRmnNodesForDifferentChains(t *testing.T) { func TestRMN_TwoMessagesOneSourceChainCursed(t *testing.T) { runRmnTestCase(t, rmnTestCase{ - name: "two messages, one source chain is cursed", + name: "two messages, one source chain is cursed the other chain was cursed but curse is revoked", passIfNoCommitAfter: 15 * time.Second, cursedSubjectsPerChain: map[int][]int{ chain1: {chain0}, }, + revokedCursedSubjectsPerChain: map[int]map[int]time.Duration{ + chain0: {globalCurse: 5 * time.Second}, // chain0 will be globally cursed and curse will be revoked later + }, homeChainConfig: homeChainConfig{ f: map[int]int{chain0: 1, chain1: 1}, }, @@ -309,6 +312,7 @@ func runRmnTestCase(t *testing.T, tc rmnTestCase) { t.Logf("Sent all messages, seqNumCommit: %v seqNumExec: %v", seqNumCommit, seqNumExec) tc.callContractsToCurseChains(ctx, t, onChainState, envWithRMN) + tc.callContractsToCurseAndRevokeCurse(ctx, t, onChainState, envWithRMN) tc.enableOracles(ctx, t, envWithRMN, disabledNodes) @@ -421,22 +425,25 @@ type rmnTestCase struct { // If set to a positive value, the test will wait for that duration and will assert that commit report was not delivered. passIfNoCommitAfter time.Duration cursedSubjectsPerChain map[int][]int - waitForExec bool - homeChainConfig homeChainConfig - remoteChainsConfig []remoteChainConfig - rmnNodes []rmnNode - messagesToSend []messageToSend + // revokedCursedSubjectsPerChain is used to revoke this specific curses after a timer expires + revokedCursedSubjectsPerChain map[int]map[int]time.Duration // chainIdx -> subjectIdx -> timer to revoke + waitForExec bool + homeChainConfig homeChainConfig + remoteChainsConfig []remoteChainConfig + rmnNodes []rmnNode + messagesToSend []messageToSend // populated fields after environment setup pf testCasePopulatedFields } type testCasePopulatedFields struct { - chainSelectors []uint64 - rmnHomeNodes []rmn_home.RMNHomeNode - rmnRemoteSigners []rmn_remote.RMNRemoteSigner - rmnHomeSourceChains []rmn_home.RMNHomeSourceChain - cursedSubjectsPerChainSel map[uint64][]uint64 + chainSelectors []uint64 + rmnHomeNodes []rmn_home.RMNHomeNode + rmnRemoteSigners []rmn_remote.RMNRemoteSigner + rmnHomeSourceChains []rmn_home.RMNHomeSourceChain + cursedSubjectsPerChainSel map[uint64][]uint64 + revokedCursedSubjectsPerChainSel map[uint64]map[uint64]time.Duration } func (tc *rmnTestCase) populateFields(t *testing.T, envWithRMN changeset.DeployedEnv, rmnCluster devenv.RMNCluster) { @@ -491,6 +498,19 @@ func (tc *rmnTestCase) populateFields(t *testing.T, envWithRMN changeset.Deploye tc.pf.cursedSubjectsPerChainSel[chainSel] = append(tc.pf.cursedSubjectsPerChainSel[chainSel], subjSel) } } + + // populate revoked cursed subjects with actual chain selectors + tc.pf.revokedCursedSubjectsPerChainSel = make(map[uint64]map[uint64]time.Duration) + for chainIdx, subjects := range tc.revokedCursedSubjectsPerChain { + chainSel := tc.pf.chainSelectors[chainIdx] + for subject, revokeAfter := range subjects { + subjSel := uint64(globalCurse) + if subject != globalCurse { + subjSel = tc.pf.chainSelectors[subject] + } + tc.pf.revokedCursedSubjectsPerChainSel[chainSel][subjSel] = revokeAfter + } + } } func (tc rmnTestCase) validate() error { @@ -638,6 +658,44 @@ func (tc rmnTestCase) callContractsToCurseChains(ctx context.Context, t *testing } } +func (tc rmnTestCase) callContractsToCurseAndRevokeCurse(ctx context.Context, t *testing.T, onChainState changeset.CCIPOnChainState, envWithRMN changeset.DeployedEnv) { + for _, remoteCfg := range tc.remoteChainsConfig { + remoteSel := tc.pf.chainSelectors[remoteCfg.chainIdx] + chState, ok := onChainState.Chains[remoteSel] + require.True(t, ok) + chain, ok := envWithRMN.Env.Chains[remoteSel] + require.True(t, ok) + + cursedSubjects, ok := tc.revokedCursedSubjectsPerChain[remoteCfg.chainIdx] + if !ok { + continue // nothing to curse on this chain + } + + for subjectDescription, revokeAfter := range cursedSubjects { + subj := reader.GlobalCurseSubject + if subjectDescription != globalCurse { + subj = chainSelectorToBytes16(tc.pf.chainSelectors[subjectDescription]) + } + t.Logf("cursing subject %d (%d)", subj, subjectDescription) + txCurse, errCurse := chState.RMNRemote.Curse(chain.DeployerKey, subj) + _, errConfirm := deployment.ConfirmIfNoError(chain, txCurse, errCurse) + require.NoError(t, errConfirm) + + go func() { + <-time.NewTimer(revokeAfter).C + t.Logf("revoking curse on subject %d (%d)", subj, subjectDescription) + txUncurse, errUncurse := chState.RMNRemote.Uncurse(chain.DeployerKey, subj) + _, errConfirm = deployment.ConfirmIfNoError(chain, txUncurse, errUncurse) + require.NoError(t, errConfirm) + }() + } + + cs, err := chState.RMNRemote.GetCursedSubjects(&bind.CallOpts{Context: ctx}) + require.NoError(t, err) + t.Logf("Cursed subjects: %v", cs) + } +} + func (tc rmnTestCase) enableOracles(ctx context.Context, t *testing.T, envWithRMN changeset.DeployedEnv, nodeIDs []string) { for _, n := range nodeIDs { _, err := envWithRMN.Env.Offchain.EnableNode(ctx, &node.EnableNodeRequest{Id: n}) From 3e9ee03617d54500a79311796fbac81a2c2aca38 Mon Sep 17 00:00:00 2001 From: dimkouv Date: Fri, 13 Dec 2024 11:26:19 +0200 Subject: [PATCH 3/3] fix assignment to nil map --- integration-tests/smoke/ccip/ccip_rmn_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration-tests/smoke/ccip/ccip_rmn_test.go b/integration-tests/smoke/ccip/ccip_rmn_test.go index 014fe3a2288..1075b28e9d3 100644 --- a/integration-tests/smoke/ccip/ccip_rmn_test.go +++ b/integration-tests/smoke/ccip/ccip_rmn_test.go @@ -508,6 +508,9 @@ func (tc *rmnTestCase) populateFields(t *testing.T, envWithRMN changeset.Deploye if subject != globalCurse { subjSel = tc.pf.chainSelectors[subject] } + if _, ok := tc.pf.revokedCursedSubjectsPerChainSel[chainSel]; !ok { + tc.pf.revokedCursedSubjectsPerChainSel[chainSel] = make(map[uint64]time.Duration) + } tc.pf.revokedCursedSubjectsPerChainSel[chainSel][subjSel] = revokeAfter } }