From 83f7413501593058443687347d0b2078410bcc1d Mon Sep 17 00:00:00 2001 From: Matthew Pendrey Date: Thu, 12 Dec 2024 16:57:18 +0000 Subject: [PATCH 01/11] Refactor workflow registry to use querykeys api (#15638) * temp disable test * wip * fix duplicate event bug - change contract reader poll query from Gte to Gt * event state sync test added and passing with fixes - single event * added methods to create different event types * pre-refactor checkpoint - up to this commit no changes have been made to wf registry - test has been added to confirm (and actually has found bugs) in current wf registry behaviour * refactor part 1: replaced querykey with querykeys - all tests passing * refactor part 2: collapse individual event query into single events query and reduce goroutine and channel usage * refactor part 3 - down to single ticker thread * tidy * enable the event ordering test that was failing in the old workflow syncer * lint remove change to try and resolve flaky eth smoke tests * log fix --- .mockery.yaml | 6 - .../workflows/syncer/workflow_syncer_test.go | 215 +++++++++++- .../workflows/syncer/contract_reader_mock.go | 302 ----------------- core/services/workflows/syncer/heap.go | 63 ---- .../workflows/syncer/workflow_registry.go | 309 +++++------------- .../syncer/workflow_registry_test.go | 234 ------------- 6 files changed, 291 insertions(+), 838 deletions(-) delete mode 100644 core/services/workflows/syncer/contract_reader_mock.go delete mode 100644 core/services/workflows/syncer/heap.go delete mode 100644 core/services/workflows/syncer/workflow_registry_test.go diff --git a/.mockery.yaml b/.mockery.yaml index dd9024cc066..5777ca1da92 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -583,12 +583,6 @@ packages: github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncer: interfaces: ORM: - ContractReader: - config: - mockname: "Mock{{ .InterfaceName }}" - filename: contract_reader_mock.go - inpackage: true - dir: "{{ .InterfaceDir }}" Handler: config: mockname: "Mock{{ .InterfaceName }}" diff --git a/core/services/relay/evm/capabilities/workflows/syncer/workflow_syncer_test.go b/core/services/relay/evm/capabilities/workflows/syncer/workflow_syncer_test.go index 3c6ee8a1d04..066e85e839f 100644 --- a/core/services/relay/evm/capabilities/workflows/syncer/workflow_syncer_test.go +++ b/core/services/relay/evm/capabilities/workflows/syncer/workflow_syncer_test.go @@ -6,7 +6,9 @@ import ( "encoding/base64" "encoding/hex" "fmt" + rand2 "math/rand/v2" "strings" + "sync" "testing" "time" @@ -31,17 +33,38 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/utils/crypto" "github.com/stretchr/testify/require" + + crypto2 "github.com/ethereum/go-ethereum/crypto" ) type testEvtHandler struct { events []syncer.Event + mux sync.Mutex } func (m *testEvtHandler) Handle(ctx context.Context, event syncer.Event) error { + m.mux.Lock() + defer m.mux.Unlock() m.events = append(m.events, event) return nil } +func (m *testEvtHandler) ClearEvents() { + m.mux.Lock() + defer m.mux.Unlock() + m.events = make([]syncer.Event, 0) +} + +func (m *testEvtHandler) GetEvents() []syncer.Event { + m.mux.Lock() + defer m.mux.Unlock() + + eventsCopy := make([]syncer.Event, len(m.events)) + copy(eventsCopy, m.events) + + return eventsCopy +} + func newTestEvtHandler() *testEvtHandler { return &testEvtHandler{ events: make([]syncer.Event, 0), @@ -68,6 +91,138 @@ func (m *testWorkflowRegistryContractLoader) LoadWorkflows(ctx context.Context, }, nil } +func Test_EventHandlerStateSync(t *testing.T) { + lggr := logger.TestLogger(t) + backendTH := testutils.NewEVMBackendTH(t) + donID := uint32(1) + + eventPollTicker := time.NewTicker(50 * time.Millisecond) + defer eventPollTicker.Stop() + + // Deploy a test workflow_registry + wfRegistryAddr, _, wfRegistryC, err := workflow_registry_wrapper.DeployWorkflowRegistry(backendTH.ContractsOwner, backendTH.Backend.Client()) + backendTH.Backend.Commit() + require.NoError(t, err) + + // setup contract state to allow the secrets to be updated + updateAllowedDONs(t, backendTH, wfRegistryC, []uint32{donID}, true) + updateAuthorizedAddress(t, backendTH, wfRegistryC, []common.Address{backendTH.ContractsOwner.From}, true) + + // Create some initial static state + numberWorkflows := 20 + for i := 0; i < numberWorkflows; i++ { + var workflowID [32]byte + _, err = rand.Read((workflowID)[:]) + require.NoError(t, err) + workflow := RegisterWorkflowCMD{ + Name: fmt.Sprintf("test-wf-%d", i), + DonID: donID, + Status: uint8(1), + SecretsURL: "someurl", + } + workflow.ID = workflowID + registerWorkflow(t, backendTH, wfRegistryC, workflow) + } + + testEventHandler := newTestEvtHandler() + loader := syncer.NewWorkflowRegistryContractLoader(lggr, wfRegistryAddr.Hex(), func(ctx context.Context, bytes []byte) (syncer.ContractReader, error) { + return backendTH.NewContractReader(ctx, t, bytes) + }, testEventHandler) + + // Create the registry + registry := syncer.NewWorkflowRegistry( + lggr, + func(ctx context.Context, bytes []byte) (syncer.ContractReader, error) { + return backendTH.NewContractReader(ctx, t, bytes) + }, + wfRegistryAddr.Hex(), + syncer.WorkflowEventPollerConfig{ + QueryCount: 20, + }, + testEventHandler, + loader, + &testDonNotifier{ + don: capabilities.DON{ + ID: donID, + }, + err: nil, + }, + syncer.WithTicker(eventPollTicker.C), + ) + + servicetest.Run(t, registry) + + require.Eventually(t, func() bool { + numEvents := len(testEventHandler.GetEvents()) + return numEvents == numberWorkflows + }, 5*time.Second, time.Second) + + for _, event := range testEventHandler.GetEvents() { + assert.Equal(t, syncer.WorkflowRegisteredEvent, event.GetEventType()) + } + + testEventHandler.ClearEvents() + + // Create different event types for a number of workflows and confirm that the event handler processes them in order + numberOfEventCycles := 50 + for i := 0; i < numberOfEventCycles; i++ { + var workflowID [32]byte + _, err = rand.Read((workflowID)[:]) + require.NoError(t, err) + workflow := RegisterWorkflowCMD{ + Name: "test-wf-register-event", + DonID: donID, + Status: uint8(1), + SecretsURL: "", + } + workflow.ID = workflowID + + // Generate events of different types with some jitter + registerWorkflow(t, backendTH, wfRegistryC, workflow) + time.Sleep(time.Millisecond * time.Duration(rand2.IntN(10))) + data := append(backendTH.ContractsOwner.From.Bytes(), []byte(workflow.Name)...) + workflowKey := crypto2.Keccak256Hash(data) + activateWorkflow(t, backendTH, wfRegistryC, workflowKey) + time.Sleep(time.Millisecond * time.Duration(rand2.IntN(10))) + pauseWorkflow(t, backendTH, wfRegistryC, workflowKey) + time.Sleep(time.Millisecond * time.Duration(rand2.IntN(10))) + var newWorkflowID [32]byte + _, err = rand.Read((newWorkflowID)[:]) + require.NoError(t, err) + updateWorkflow(t, backendTH, wfRegistryC, workflowKey, newWorkflowID, workflow.BinaryURL+"2", workflow.ConfigURL, workflow.SecretsURL) + time.Sleep(time.Millisecond * time.Duration(rand2.IntN(10))) + deleteWorkflow(t, backendTH, wfRegistryC, workflowKey) + } + + // Confirm the expected number of events are received in the correct order + require.Eventually(t, func() bool { + events := testEventHandler.GetEvents() + numEvents := len(events) + expectedNumEvents := 5 * numberOfEventCycles + + if numEvents == expectedNumEvents { + // verify the events are the expected types in the expected order + for idx, event := range events { + switch idx % 5 { + case 0: + assert.Equal(t, syncer.WorkflowRegisteredEvent, event.GetEventType()) + case 1: + assert.Equal(t, syncer.WorkflowActivatedEvent, event.GetEventType()) + case 2: + assert.Equal(t, syncer.WorkflowPausedEvent, event.GetEventType()) + case 3: + assert.Equal(t, syncer.WorkflowUpdatedEvent, event.GetEventType()) + case 4: + assert.Equal(t, syncer.WorkflowDeletedEvent, event.GetEventType()) + } + } + return true + } + + return false + }, 50*time.Second, time.Second) +} + func Test_InitialStateSync(t *testing.T) { lggr := logger.TestLogger(t) backendTH := testutils.NewEVMBackendTH(t) @@ -128,10 +283,10 @@ func Test_InitialStateSync(t *testing.T) { servicetest.Run(t, worker) require.Eventually(t, func() bool { - return len(testEventHandler.events) == numberWorkflows + return len(testEventHandler.GetEvents()) == numberWorkflows }, 5*time.Second, time.Second) - for _, event := range testEventHandler.events { + for _, event := range testEventHandler.GetEvents() { assert.Equal(t, syncer.WorkflowRegisteredEvent, event.GetEventType()) } } @@ -497,3 +652,59 @@ func requestForceUpdateSecrets( th.Backend.Commit() th.Backend.Commit() } + +func activateWorkflow( + t *testing.T, + th *testutils.EVMBackendTH, + wfRegC *workflow_registry_wrapper.WorkflowRegistry, + workflowKey [32]byte, +) { + t.Helper() + _, err := wfRegC.ActivateWorkflow(th.ContractsOwner, workflowKey) + require.NoError(t, err, "failed to activate workflow") + th.Backend.Commit() + th.Backend.Commit() + th.Backend.Commit() +} + +func pauseWorkflow( + t *testing.T, + th *testutils.EVMBackendTH, + wfRegC *workflow_registry_wrapper.WorkflowRegistry, + workflowKey [32]byte, +) { + t.Helper() + _, err := wfRegC.PauseWorkflow(th.ContractsOwner, workflowKey) + require.NoError(t, err, "failed to pause workflow") + th.Backend.Commit() + th.Backend.Commit() + th.Backend.Commit() +} + +func deleteWorkflow( + t *testing.T, + th *testutils.EVMBackendTH, + wfRegC *workflow_registry_wrapper.WorkflowRegistry, + workflowKey [32]byte, +) { + t.Helper() + _, err := wfRegC.DeleteWorkflow(th.ContractsOwner, workflowKey) + require.NoError(t, err, "failed to delete workflow") + th.Backend.Commit() + th.Backend.Commit() + th.Backend.Commit() +} + +func updateWorkflow( + t *testing.T, + th *testutils.EVMBackendTH, + wfRegC *workflow_registry_wrapper.WorkflowRegistry, + workflowKey [32]byte, newWorkflowID [32]byte, binaryURL string, configURL string, secretsURL string, +) { + t.Helper() + _, err := wfRegC.UpdateWorkflow(th.ContractsOwner, workflowKey, newWorkflowID, binaryURL, configURL, secretsURL) + require.NoError(t, err, "failed to update workflow") + th.Backend.Commit() + th.Backend.Commit() + th.Backend.Commit() +} diff --git a/core/services/workflows/syncer/contract_reader_mock.go b/core/services/workflows/syncer/contract_reader_mock.go deleted file mode 100644 index e6e7c8385f5..00000000000 --- a/core/services/workflows/syncer/contract_reader_mock.go +++ /dev/null @@ -1,302 +0,0 @@ -// Code generated by mockery v2.46.3. DO NOT EDIT. - -package syncer - -import ( - context "context" - - query "github.com/smartcontractkit/chainlink-common/pkg/types/query" - primitives "github.com/smartcontractkit/chainlink-common/pkg/types/query/primitives" - mock "github.com/stretchr/testify/mock" - - types "github.com/smartcontractkit/chainlink-common/pkg/types" -) - -// MockContractReader is an autogenerated mock type for the ContractReader type -type MockContractReader struct { - mock.Mock -} - -type MockContractReader_Expecter struct { - mock *mock.Mock -} - -func (_m *MockContractReader) EXPECT() *MockContractReader_Expecter { - return &MockContractReader_Expecter{mock: &_m.Mock} -} - -// Bind provides a mock function with given fields: _a0, _a1 -func (_m *MockContractReader) Bind(_a0 context.Context, _a1 []types.BoundContract) error { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for Bind") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, []types.BoundContract) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// MockContractReader_Bind_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Bind' -type MockContractReader_Bind_Call struct { - *mock.Call -} - -// Bind is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 []types.BoundContract -func (_e *MockContractReader_Expecter) Bind(_a0 interface{}, _a1 interface{}) *MockContractReader_Bind_Call { - return &MockContractReader_Bind_Call{Call: _e.mock.On("Bind", _a0, _a1)} -} - -func (_c *MockContractReader_Bind_Call) Run(run func(_a0 context.Context, _a1 []types.BoundContract)) *MockContractReader_Bind_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].([]types.BoundContract)) - }) - return _c -} - -func (_c *MockContractReader_Bind_Call) Return(_a0 error) *MockContractReader_Bind_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *MockContractReader_Bind_Call) RunAndReturn(run func(context.Context, []types.BoundContract) error) *MockContractReader_Bind_Call { - _c.Call.Return(run) - return _c -} - -// Close provides a mock function with given fields: -func (_m *MockContractReader) Close() error { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Close") - } - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// MockContractReader_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' -type MockContractReader_Close_Call struct { - *mock.Call -} - -// Close is a helper method to define mock.On call -func (_e *MockContractReader_Expecter) Close() *MockContractReader_Close_Call { - return &MockContractReader_Close_Call{Call: _e.mock.On("Close")} -} - -func (_c *MockContractReader_Close_Call) Run(run func()) *MockContractReader_Close_Call { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *MockContractReader_Close_Call) Return(_a0 error) *MockContractReader_Close_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *MockContractReader_Close_Call) RunAndReturn(run func() error) *MockContractReader_Close_Call { - _c.Call.Return(run) - return _c -} - -// GetLatestValueWithHeadData provides a mock function with given fields: ctx, readName, confidenceLevel, params, returnVal -func (_m *MockContractReader) GetLatestValueWithHeadData(ctx context.Context, readName string, confidenceLevel primitives.ConfidenceLevel, params any, returnVal any) (*types.Head, error) { - ret := _m.Called(ctx, readName, confidenceLevel, params, returnVal) - - if len(ret) == 0 { - panic("no return value specified for GetLatestValueWithHeadData") - } - - var r0 *types.Head - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, primitives.ConfidenceLevel, any, any) (*types.Head, error)); ok { - return rf(ctx, readName, confidenceLevel, params, returnVal) - } - if rf, ok := ret.Get(0).(func(context.Context, string, primitives.ConfidenceLevel, any, any) *types.Head); ok { - r0 = rf(ctx, readName, confidenceLevel, params, returnVal) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*types.Head) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, primitives.ConfidenceLevel, any, any) error); ok { - r1 = rf(ctx, readName, confidenceLevel, params, returnVal) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// MockContractReader_GetLatestValueWithHeadData_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLatestValueWithHeadData' -type MockContractReader_GetLatestValueWithHeadData_Call struct { - *mock.Call -} - -// GetLatestValueWithHeadData is a helper method to define mock.On call -// - ctx context.Context -// - readName string -// - confidenceLevel primitives.ConfidenceLevel -// - params any -// - returnVal any -func (_e *MockContractReader_Expecter) GetLatestValueWithHeadData(ctx interface{}, readName interface{}, confidenceLevel interface{}, params interface{}, returnVal interface{}) *MockContractReader_GetLatestValueWithHeadData_Call { - return &MockContractReader_GetLatestValueWithHeadData_Call{Call: _e.mock.On("GetLatestValueWithHeadData", ctx, readName, confidenceLevel, params, returnVal)} -} - -func (_c *MockContractReader_GetLatestValueWithHeadData_Call) Run(run func(ctx context.Context, readName string, confidenceLevel primitives.ConfidenceLevel, params any, returnVal any)) *MockContractReader_GetLatestValueWithHeadData_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string), args[2].(primitives.ConfidenceLevel), args[3].(any), args[4].(any)) - }) - return _c -} - -func (_c *MockContractReader_GetLatestValueWithHeadData_Call) Return(head *types.Head, err error) *MockContractReader_GetLatestValueWithHeadData_Call { - _c.Call.Return(head, err) - return _c -} - -func (_c *MockContractReader_GetLatestValueWithHeadData_Call) RunAndReturn(run func(context.Context, string, primitives.ConfidenceLevel, any, any) (*types.Head, error)) *MockContractReader_GetLatestValueWithHeadData_Call { - _c.Call.Return(run) - return _c -} - -// QueryKey provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 -func (_m *MockContractReader) QueryKey(_a0 context.Context, _a1 types.BoundContract, _a2 query.KeyFilter, _a3 query.LimitAndSort, _a4 any) ([]types.Sequence, error) { - ret := _m.Called(_a0, _a1, _a2, _a3, _a4) - - if len(ret) == 0 { - panic("no return value specified for QueryKey") - } - - var r0 []types.Sequence - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, types.BoundContract, query.KeyFilter, query.LimitAndSort, any) ([]types.Sequence, error)); ok { - return rf(_a0, _a1, _a2, _a3, _a4) - } - if rf, ok := ret.Get(0).(func(context.Context, types.BoundContract, query.KeyFilter, query.LimitAndSort, any) []types.Sequence); ok { - r0 = rf(_a0, _a1, _a2, _a3, _a4) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]types.Sequence) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, types.BoundContract, query.KeyFilter, query.LimitAndSort, any) error); ok { - r1 = rf(_a0, _a1, _a2, _a3, _a4) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// MockContractReader_QueryKey_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'QueryKey' -type MockContractReader_QueryKey_Call struct { - *mock.Call -} - -// QueryKey is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 types.BoundContract -// - _a2 query.KeyFilter -// - _a3 query.LimitAndSort -// - _a4 any -func (_e *MockContractReader_Expecter) QueryKey(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}, _a4 interface{}) *MockContractReader_QueryKey_Call { - return &MockContractReader_QueryKey_Call{Call: _e.mock.On("QueryKey", _a0, _a1, _a2, _a3, _a4)} -} - -func (_c *MockContractReader_QueryKey_Call) Run(run func(_a0 context.Context, _a1 types.BoundContract, _a2 query.KeyFilter, _a3 query.LimitAndSort, _a4 any)) *MockContractReader_QueryKey_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(types.BoundContract), args[2].(query.KeyFilter), args[3].(query.LimitAndSort), args[4].(any)) - }) - return _c -} - -func (_c *MockContractReader_QueryKey_Call) Return(_a0 []types.Sequence, _a1 error) *MockContractReader_QueryKey_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *MockContractReader_QueryKey_Call) RunAndReturn(run func(context.Context, types.BoundContract, query.KeyFilter, query.LimitAndSort, any) ([]types.Sequence, error)) *MockContractReader_QueryKey_Call { - _c.Call.Return(run) - return _c -} - -// Start provides a mock function with given fields: ctx -func (_m *MockContractReader) Start(ctx context.Context) error { - ret := _m.Called(ctx) - - if len(ret) == 0 { - panic("no return value specified for Start") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context) error); ok { - r0 = rf(ctx) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// MockContractReader_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' -type MockContractReader_Start_Call struct { - *mock.Call -} - -// Start is a helper method to define mock.On call -// - ctx context.Context -func (_e *MockContractReader_Expecter) Start(ctx interface{}) *MockContractReader_Start_Call { - return &MockContractReader_Start_Call{Call: _e.mock.On("Start", ctx)} -} - -func (_c *MockContractReader_Start_Call) Run(run func(ctx context.Context)) *MockContractReader_Start_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *MockContractReader_Start_Call) Return(_a0 error) *MockContractReader_Start_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *MockContractReader_Start_Call) RunAndReturn(run func(context.Context) error) *MockContractReader_Start_Call { - _c.Call.Return(run) - return _c -} - -// NewMockContractReader creates a new instance of MockContractReader. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewMockContractReader(t interface { - mock.TestingT - Cleanup(func()) -}) *MockContractReader { - mock := &MockContractReader{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/core/services/workflows/syncer/heap.go b/core/services/workflows/syncer/heap.go deleted file mode 100644 index 061293928a3..00000000000 --- a/core/services/workflows/syncer/heap.go +++ /dev/null @@ -1,63 +0,0 @@ -package syncer - -import "container/heap" - -type Heap interface { - // Push adds a new item to the heap. - Push(x WorkflowRegistryEventResponse) - - // Pop removes the smallest item from the heap and returns it. - Pop() WorkflowRegistryEventResponse - - // Len returns the number of items in the heap. - Len() int -} - -// publicHeap is a wrapper around the heap.Interface that exposes the Push and Pop methods. -type publicHeap[T any] struct { - heap heap.Interface -} - -func (h *publicHeap[T]) Push(x T) { - heap.Push(h.heap, x) -} - -func (h *publicHeap[T]) Pop() T { - return heap.Pop(h.heap).(T) -} - -func (h *publicHeap[T]) Len() int { - return h.heap.Len() -} - -// blockHeightHeap is a heap.Interface that sorts WorkflowRegistryEventResponses by block height. -type blockHeightHeap []WorkflowRegistryEventResponse - -// newBlockHeightHeap returns an initialized heap that sorts WorkflowRegistryEventResponses by block height. -func newBlockHeightHeap() Heap { - h := blockHeightHeap(make([]WorkflowRegistryEventResponse, 0)) - heap.Init(&h) - return &publicHeap[WorkflowRegistryEventResponse]{heap: &h} -} - -func (h *blockHeightHeap) Len() int { return len(*h) } - -func (h *blockHeightHeap) Less(i, j int) bool { - return (*h)[i].Event.Head.Height < (*h)[j].Event.Head.Height -} - -func (h *blockHeightHeap) Swap(i, j int) { - (*h)[i], (*h)[j] = (*h)[j], (*h)[i] -} - -func (h *blockHeightHeap) Push(x any) { - *h = append(*h, x.(WorkflowRegistryEventResponse)) -} - -func (h *blockHeightHeap) Pop() any { - old := *h - n := len(old) - x := old[n-1] - *h = old[0 : n-1] - return x -} diff --git a/core/services/workflows/syncer/workflow_registry.go b/core/services/workflows/syncer/workflow_registry.go index 75fcc9735ad..223fbe8e758 100644 --- a/core/services/workflows/syncer/workflow_registry.go +++ b/core/services/workflows/syncer/workflow_registry.go @@ -5,13 +5,14 @@ import ( "encoding/hex" "encoding/json" "fmt" + "iter" "sync" "time" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/services" - types "github.com/smartcontractkit/chainlink-common/pkg/types" - query "github.com/smartcontractkit/chainlink-common/pkg/types/query" + "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/chainlink-common/pkg/types/query" "github.com/smartcontractkit/chainlink-common/pkg/types/query/primitives" "github.com/smartcontractkit/chainlink-common/pkg/values" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/workflow/generated/workflow_registry_wrapper" @@ -90,19 +91,19 @@ type WorkflowLoadConfig struct { // FetcherFunc is an abstraction for fetching the contents stored at a URL. type FetcherFunc func(ctx context.Context, url string) ([]byte, error) -type ContractReaderFactory interface { - NewContractReader(context.Context, []byte) (types.ContractReader, error) -} - // ContractReader is a subset of types.ContractReader defined locally to enable mocking. type ContractReader interface { Start(ctx context.Context) error Close() error Bind(context.Context, []types.BoundContract) error - QueryKey(context.Context, types.BoundContract, query.KeyFilter, query.LimitAndSort, any) ([]types.Sequence, error) + QueryKeys(ctx context.Context, keyQueries []types.ContractKeyFilter, limitAndSort query.LimitAndSort) (iter.Seq2[string, types.Sequence], error) GetLatestValueWithHeadData(ctx context.Context, readName string, confidenceLevel primitives.ConfidenceLevel, params any, returnVal any) (head *types.Head, err error) } +type ContractReaderFactory interface { + NewContractReader(context.Context, []byte) (types.ContractReader, error) +} + // WorkflowRegistrySyncer is the public interface of the package. type WorkflowRegistrySyncer interface { services.Service @@ -128,21 +129,11 @@ type workflowRegistry struct { newContractReaderFn newContractReaderFn - eventPollerCfg WorkflowEventPollerConfig - eventTypes []WorkflowRegistryEventType - - // eventsCh is read by the handler and each event is handled once received. - eventsCh chan WorkflowRegistryEventResponse + eventPollerCfg WorkflowEventPollerConfig + eventTypes []WorkflowRegistryEventType handler evtHandler initialWorkflowsStateLoader initialWorkflowsStateLoader - // batchCh is a channel that receives batches of events from the contract query goroutines. - batchCh chan []WorkflowRegistryEventResponse - - // heap is a min heap that merges batches of events from the contract query goroutines. The - // default min heap is sorted by block height. - heap Heap - workflowDonNotifier donNotifier reader ContractReader @@ -197,11 +188,8 @@ func NewWorkflowRegistry( newContractReaderFn: newContractReaderFn, workflowRegistryAddress: addr, eventPollerCfg: eventPollerConfig, - heap: newBlockHeightHeap(), stopCh: make(services.StopChan), eventTypes: ets, - eventsCh: make(chan WorkflowRegistryEventResponse), - batchCh: make(chan []WorkflowRegistryEventResponse, len(ets)), handler: handler, initialWorkflowsStateLoader: initialWorkflowsStateLoader, workflowDonNotifier: workflowDonNotifier, @@ -238,15 +226,13 @@ func (w *workflowRegistry) Start(_ context.Context) error { return } - w.syncEventsLoop(ctx, loadWorkflowsHead.Height) - }() - - w.wg.Add(1) - go func() { - defer w.wg.Done() - defer cancel() + reader, err := w.getContractReader(ctx) + if err != nil { + w.lggr.Criticalf("contract reader unavailable : %s", err) + return + } - w.handlerLoop(ctx) + w.readRegistryEvents(ctx, reader, loadWorkflowsHead.Height) }() return nil @@ -273,135 +259,82 @@ func (w *workflowRegistry) Name() string { return name } -// handlerLoop handles the events that are emitted by the contract. -func (w *workflowRegistry) handlerLoop(ctx context.Context) { +// readRegistryEvents polls the contract for events and send them to the events channel. +func (w *workflowRegistry) readRegistryEvents(ctx context.Context, reader ContractReader, lastReadBlockNumber string) { + ticker := w.getTicker() + + var keyQueries = make([]types.ContractKeyFilter, 0, len(w.eventTypes)) + for _, et := range w.eventTypes { + var logData values.Value + keyQueries = append(keyQueries, types.ContractKeyFilter{ + KeyFilter: query.KeyFilter{ + Key: string(et), + Expressions: []query.Expression{ + query.Confidence(primitives.Finalized), + query.Block(lastReadBlockNumber, primitives.Gt), + }, + }, + Contract: types.BoundContract{ + Name: WorkflowRegistryContractName, + Address: w.workflowRegistryAddress, + }, + SequenceDataType: &logData, + }) + } + + cursor := "" for { select { case <-ctx.Done(): return - case resp, open := <-w.eventsCh: - if !open { - return + case <-ticker: + limitAndSort := query.LimitAndSort{ + SortBy: []query.SortBy{query.NewSortByTimestamp(query.Asc)}, + Limit: query.Limit{Count: w.eventPollerCfg.QueryCount}, } - - if resp.Err != nil || resp.Event == nil { - w.lggr.Errorw("failed to handle event", "err", resp.Err) - continue + if cursor != "" { + limitAndSort.Limit = query.CursorLimit(cursor, query.CursorFollowing, w.eventPollerCfg.QueryCount) } - event := resp.Event - w.lggr.Debugf("handling event: %+v", event) - if err := w.handler.Handle(ctx, *event); err != nil { - w.lggr.Errorw("failed to handle event", "event", event, "err", err) + logsIter, err := reader.QueryKeys(ctx, keyQueries, limitAndSort) + if err != nil { + w.lggr.Errorw("failed to query keys", "err", err) continue } - } - } -} -// syncEventsLoop polls the contract for events and passes them to a channel for handling. -func (w *workflowRegistry) syncEventsLoop(ctx context.Context, lastReadBlockNumber string) { - var ( - // sendLog is a helper that sends a WorkflowRegistryEventResponse to the eventsCh in a - // blocking way that will send the response or be canceled. - sendLog = func(resp WorkflowRegistryEventResponse) { - select { - case w.eventsCh <- resp: - case <-ctx.Done(): + var logs []sequenceWithEventType + for eventType, log := range logsIter { + logs = append(logs, sequenceWithEventType{ + Sequence: log, + EventType: WorkflowRegistryEventType(eventType), + }) } - } - - ticker = w.getTicker() - - signals = make(map[WorkflowRegistryEventType]chan struct{}, 0) - ) - - // critical failure if there is no reader, the loop will exit and the parent context will be - // canceled. - reader, err := w.getContractReader(ctx) - if err != nil { - w.lggr.Criticalf("contract reader unavailable : %s", err) - return - } - - // fan out and query for each event type - for i := 0; i < len(w.eventTypes); i++ { - signal := make(chan struct{}, 1) - signals[w.eventTypes[i]] = signal - w.wg.Add(1) - go func() { - defer w.wg.Done() - - queryEvent( - ctx, - signal, - w.lggr, - reader, - lastReadBlockNumber, - queryEventConfig{ - ContractName: WorkflowRegistryContractName, - ContractAddress: w.workflowRegistryAddress, - WorkflowEventPollerConfig: w.eventPollerCfg, - }, - w.eventTypes[i], - w.batchCh, - ) - }() - } + w.lggr.Debugw("QueryKeys called", "logs", len(logs), "eventTypes", w.eventTypes, "lastReadBlockNumber", lastReadBlockNumber, "logCursor", cursor) - // Periodically send a signal to all the queryEvent goroutines to query the contract - for { - select { - case <-ctx.Done(): - return - case <-ticker: - w.lggr.Debugw("Syncing with WorkflowRegistry") - // for each event type, send a signal for it to execute a query and produce a new - // batch of event logs - for i := 0; i < len(w.eventTypes); i++ { - signal := signals[w.eventTypes[i]] - select { - case signal <- struct{}{}: - case <-ctx.Done(): - return - } + // ChainReader QueryKey API provides logs including the cursor value and not + // after the cursor value. If the response only consists of the log corresponding + // to the cursor and no log after it, then we understand that there are no new + // logs + if len(logs) == 1 && logs[0].Sequence.Cursor == cursor { + w.lggr.Infow("No new logs since", "cursor", cursor) + continue } - // block on fan-in until all fetched event logs are sent to the handlers - w.orderAndSend( - ctx, - len(w.eventTypes), - w.batchCh, - sendLog, - ) - } - } -} + var events []WorkflowRegistryEventResponse + for _, log := range logs { + if log.Sequence.Cursor == cursor { + continue + } -// orderAndSend reads n batches from the batch channel, heapifies all the batches then dequeues -// the min heap via the sendLog function. -func (w *workflowRegistry) orderAndSend( - ctx context.Context, - batchCount int, - batchCh <-chan []WorkflowRegistryEventResponse, - sendLog func(WorkflowRegistryEventResponse), -) { - for { - select { - case <-ctx.Done(): - return - case batch := <-batchCh: - for _, response := range batch { - w.heap.Push(response) + events = append(events, toWorkflowRegistryEventResponse(log.Sequence, log.EventType, w.lggr)) + cursor = log.Sequence.Cursor } - batchCount-- - // If we have received responses for all the events, then we can drain the heap. - if batchCount == 0 { - for w.heap.Len() > 0 { - sendLog(w.heap.Pop()) + for _, event := range events { + err := w.handler.Handle(ctx, event.Event) + if err != nil { + w.lggr.Errorw("failed to handle event", "err", err) } - return } } } @@ -437,95 +370,9 @@ func (w *workflowRegistry) getContractReader(ctx context.Context) (ContractReade return w.reader, nil } -type queryEventConfig struct { - ContractName string - ContractAddress string - WorkflowEventPollerConfig -} - -// queryEvent queries the contract for events of the given type on each tick from the ticker. -// Sends a batch of event logs to the batch channel. The batch represents all the -// event logs read since the last query. Loops until the context is canceled. -func queryEvent( - ctx context.Context, - ticker <-chan struct{}, - lggr logger.Logger, - reader ContractReader, - lastReadBlockNumber string, - cfg queryEventConfig, - et WorkflowRegistryEventType, - batchCh chan<- []WorkflowRegistryEventResponse, -) { - // create query - var ( - logData values.Value - cursor = "" - limitAndSort = query.LimitAndSort{ - SortBy: []query.SortBy{query.NewSortByTimestamp(query.Asc)}, - Limit: query.Limit{Count: cfg.QueryCount}, - } - bc = types.BoundContract{ - Name: cfg.ContractName, - Address: cfg.ContractAddress, - } - ) - - // Loop until canceled - for { - select { - case <-ctx.Done(): - return - case <-ticker: - responseBatch := []WorkflowRegistryEventResponse{} - - if cursor != "" { - limitAndSort.Limit = query.CursorLimit(cursor, query.CursorFollowing, cfg.QueryCount) - } - - logs, err := reader.QueryKey( - ctx, - bc, - query.KeyFilter{ - Key: string(et), - Expressions: []query.Expression{ - query.Confidence(primitives.Finalized), - query.Block(lastReadBlockNumber, primitives.Gte), - }, - }, - limitAndSort, - &logData, - ) - lcursor := cursor - if lcursor == "" { - lcursor = "empty" - } - lggr.Debugw("QueryKeys called", "logs", len(logs), "eventType", et, "lastReadBlockNumber", lastReadBlockNumber, "logCursor", lcursor) - - if err != nil { - lggr.Errorw("QueryKey failure", "err", err) - continue - } - - // ChainReader QueryKey API provides logs including the cursor value and not - // after the cursor value. If the response only consists of the log corresponding - // to the cursor and no log after it, then we understand that there are no new - // logs - if len(logs) == 1 && logs[0].Cursor == cursor { - lggr.Infow("No new logs since", "cursor", cursor) - continue - } - - for _, log := range logs { - if log.Cursor == cursor { - continue - } - - responseBatch = append(responseBatch, toWorkflowRegistryEventResponse(log, et, lggr)) - cursor = log.Cursor - } - batchCh <- responseBatch - } - } +type sequenceWithEventType struct { + Sequence types.Sequence + EventType WorkflowRegistryEventType } func getWorkflowRegistryEventReader( @@ -681,7 +528,7 @@ func (l *workflowRegistryContractLoader) LoadWorkflows(ctx context.Context, don var workflows GetWorkflowMetadataListByDONReturnVal headAtLastRead, err = contractReader.GetLatestValueWithHeadData(ctx, readIdentifier, primitives.Finalized, params, &workflows) if err != nil { - return nil, fmt.Errorf("failed to get workflow metadata for don %w", err) + return nil, fmt.Errorf("failed to get lastest value with head data %w", err) } l.lggr.Debugw("Rehydrating existing workflows", "len", len(workflows.WorkflowMetadataList)) diff --git a/core/services/workflows/syncer/workflow_registry_test.go b/core/services/workflows/syncer/workflow_registry_test.go deleted file mode 100644 index 621d3d123d5..00000000000 --- a/core/services/workflows/syncer/workflow_registry_test.go +++ /dev/null @@ -1,234 +0,0 @@ -package syncer - -import ( - "context" - "encoding/hex" - "testing" - "time" - - "github.com/stretchr/testify/mock" - - "github.com/jonboulle/clockwork" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities" - "github.com/smartcontractkit/chainlink-common/pkg/custmsg" - "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" - types "github.com/smartcontractkit/chainlink-common/pkg/types" - query "github.com/smartcontractkit/chainlink-common/pkg/types/query" - "github.com/smartcontractkit/chainlink-common/pkg/types/query/primitives" - "github.com/smartcontractkit/chainlink-common/pkg/values" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/workflowkey" - "github.com/smartcontractkit/chainlink/v2/core/utils/crypto" - "github.com/smartcontractkit/chainlink/v2/core/utils/matches" - - "github.com/stretchr/testify/require" -) - -type testDonNotifier struct { - don capabilities.DON - err error -} - -func (t *testDonNotifier) WaitForDon(ctx context.Context) (capabilities.DON, error) { - return t.don, t.err -} - -func Test_Workflow_Registry_Syncer(t *testing.T) { - var ( - giveContents = "contents" - wantContents = "updated contents" - contractAddress = "0xdeadbeef" - giveCfg = WorkflowEventPollerConfig{ - QueryCount: 20, - } - giveURL = "http://example.com" - giveHash, err = crypto.Keccak256([]byte(giveURL)) - - giveLog = types.Sequence{ - Data: map[string]any{ - "SecretsURLHash": giveHash, - "Owner": "0xowneraddr", - }, - Cursor: "cursor", - } - ) - - require.NoError(t, err) - - var ( - lggr = logger.TestLogger(t) - db = pgtest.NewSqlxDB(t) - orm = &orm{ds: db, lggr: lggr} - ctx, cancel = context.WithCancel(testutils.Context(t)) - reader = NewMockContractReader(t) - emitter = custmsg.NewLabeler() - gateway = func(_ context.Context, _ string) ([]byte, error) { - return []byte(wantContents), nil - } - ticker = make(chan time.Time) - - handler = NewEventHandler(lggr, orm, gateway, nil, nil, - emitter, clockwork.NewFakeClock(), workflowkey.Key{}) - loader = NewWorkflowRegistryContractLoader(lggr, contractAddress, func(ctx context.Context, bytes []byte) (ContractReader, error) { - return reader, nil - }, handler) - - worker = NewWorkflowRegistry(lggr, func(ctx context.Context, bytes []byte) (ContractReader, error) { - return reader, nil - }, contractAddress, - WorkflowEventPollerConfig{ - QueryCount: 20, - }, handler, loader, - &testDonNotifier{ - don: capabilities.DON{ - ID: 1, - }, - err: nil, - }, - WithTicker(ticker)) - ) - - // Cleanup the worker - defer cancel() - - // Seed the DB with an original entry - _, err = orm.Create(ctx, giveURL, hex.EncodeToString(giveHash), giveContents) - require.NoError(t, err) - - // Mock out the contract reader query - reader.EXPECT().QueryKey( - matches.AnyContext, - types.BoundContract{ - Name: WorkflowRegistryContractName, - Address: contractAddress, - }, - query.KeyFilter{ - Key: string(ForceUpdateSecretsEvent), - Expressions: []query.Expression{ - query.Confidence(primitives.Finalized), - query.Block("0", primitives.Gte), - }, - }, - query.LimitAndSort{ - SortBy: []query.SortBy{query.NewSortByTimestamp(query.Asc)}, - Limit: query.Limit{Count: giveCfg.QueryCount}, - }, - new(values.Value), - ).Return([]types.Sequence{giveLog}, nil) - reader.EXPECT().QueryKey( - matches.AnyContext, - types.BoundContract{ - Name: WorkflowRegistryContractName, - Address: contractAddress, - }, - query.KeyFilter{ - Key: string(WorkflowPausedEvent), - Expressions: []query.Expression{ - query.Confidence(primitives.Finalized), - query.Block("0", primitives.Gte), - }, - }, - query.LimitAndSort{ - SortBy: []query.SortBy{query.NewSortByTimestamp(query.Asc)}, - Limit: query.Limit{Count: giveCfg.QueryCount}, - }, - new(values.Value), - ).Return([]types.Sequence{}, nil) - reader.EXPECT().QueryKey( - matches.AnyContext, - types.BoundContract{ - Name: WorkflowRegistryContractName, - Address: contractAddress, - }, - query.KeyFilter{ - Key: string(WorkflowDeletedEvent), - Expressions: []query.Expression{ - query.Confidence(primitives.Finalized), - query.Block("0", primitives.Gte), - }, - }, - query.LimitAndSort{ - SortBy: []query.SortBy{query.NewSortByTimestamp(query.Asc)}, - Limit: query.Limit{Count: giveCfg.QueryCount}, - }, - new(values.Value), - ).Return([]types.Sequence{}, nil) - reader.EXPECT().QueryKey( - matches.AnyContext, - types.BoundContract{ - Name: WorkflowRegistryContractName, - Address: contractAddress, - }, - query.KeyFilter{ - Key: string(WorkflowActivatedEvent), - Expressions: []query.Expression{ - query.Confidence(primitives.Finalized), - query.Block("0", primitives.Gte), - }, - }, - query.LimitAndSort{ - SortBy: []query.SortBy{query.NewSortByTimestamp(query.Asc)}, - Limit: query.Limit{Count: giveCfg.QueryCount}, - }, - new(values.Value), - ).Return([]types.Sequence{}, nil) - reader.EXPECT().QueryKey( - matches.AnyContext, - types.BoundContract{ - Name: WorkflowRegistryContractName, - Address: contractAddress, - }, - query.KeyFilter{ - Key: string(WorkflowUpdatedEvent), - Expressions: []query.Expression{ - query.Confidence(primitives.Finalized), - query.Block("0", primitives.Gte), - }, - }, - query.LimitAndSort{ - SortBy: []query.SortBy{query.NewSortByTimestamp(query.Asc)}, - Limit: query.Limit{Count: giveCfg.QueryCount}, - }, - new(values.Value), - ).Return([]types.Sequence{}, nil) - reader.EXPECT().QueryKey( - matches.AnyContext, - types.BoundContract{ - Name: WorkflowRegistryContractName, - Address: contractAddress, - }, - query.KeyFilter{ - Key: string(WorkflowRegisteredEvent), - Expressions: []query.Expression{ - query.Confidence(primitives.Finalized), - query.Block("0", primitives.Gte), - }, - }, - query.LimitAndSort{ - SortBy: []query.SortBy{query.NewSortByTimestamp(query.Asc)}, - Limit: query.Limit{Count: giveCfg.QueryCount}, - }, - new(values.Value), - ).Return([]types.Sequence{}, nil) - reader.EXPECT().GetLatestValueWithHeadData(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&types.Head{ - Height: "0", - }, nil) - reader.EXPECT().Start(mock.Anything).Return(nil) - reader.EXPECT().Bind(mock.Anything, mock.Anything).Return(nil) - - // Go run the worker - servicetest.Run(t, worker) - - // Send a tick to start a query - ticker <- time.Now() - - // Require the secrets contents to eventually be updated - require.Eventually(t, func() bool { - secrets, err := orm.GetContents(ctx, giveURL) - require.NoError(t, err) - return secrets == wantContents - }, 5*time.Second, time.Second) -} From eaeb2ebe7bfc53572655be322b793f0bf9556e1e Mon Sep 17 00:00:00 2001 From: krehermann <16602512+krehermann@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:26:36 -0700 Subject: [PATCH 02/11] [KS-616] asset job updates (#15573) * fix error handling * add test * idempotent registry * add changeset * fix tests * isolate fix to mercury; more tests --- .changeset/big-camels-report.md | 5 + core/services/ocr2/plugins/mercury/plugin.go | 89 ++++++++---- .../ocr2/plugins/mercury/plugin_test.go | 133 ++++++++++++++++-- plugins/registrar.go | 2 +- 4 files changed, 190 insertions(+), 39 deletions(-) create mode 100644 .changeset/big-camels-report.md diff --git a/.changeset/big-camels-report.md b/.changeset/big-camels-report.md new file mode 100644 index 00000000000..f81f66b9138 --- /dev/null +++ b/.changeset/big-camels-report.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +#bugfix fix non-idempotent loopp registry.Register diff --git a/core/services/ocr2/plugins/mercury/plugin.go b/core/services/ocr2/plugins/mercury/plugin.go index 8a4101804dd..b0983e55c89 100644 --- a/core/services/ocr2/plugins/mercury/plugin.go +++ b/core/services/ocr2/plugins/mercury/plugin.go @@ -1,6 +1,7 @@ package mercury import ( + "context" "encoding/json" "fmt" "os/exec" @@ -79,14 +80,13 @@ func NewServices( return nil, errors.New("expected job to have a non-nil PipelineSpec") } - var err error var pluginConfig config.PluginConfig if len(jb.OCR2OracleSpec.PluginConfig) == 0 { if !enableTriggerCapability { return nil, fmt.Errorf("at least one transmission option must be configured") } } else { - err = json.Unmarshal(jb.OCR2OracleSpec.PluginConfig.Bytes(), &pluginConfig) + err := json.Unmarshal(jb.OCR2OracleSpec.PluginConfig.Bytes(), &pluginConfig) if err != nil { return nil, errors.WithStack(err) } @@ -101,8 +101,8 @@ func NewServices( // encapsulate all the subservices and ensure we close them all if any fail to start srvs := []job.ServiceCtx{ocr2Provider} abort := func() { - if err = services.MultiCloser(srvs).Close(); err != nil { - lggr.Errorw("Error closing unused services", "err", err) + if cerr := services.MultiCloser(srvs).Close(); cerr != nil { + lggr.Errorw("Error closing unused services", "err", cerr) } } saver := ocrcommon.NewResultRunSaver(pipelineRunner, lggr, cfg.MaxSuccessfulRuns(), cfg.ResultWriteQueueDepth()) @@ -112,6 +112,7 @@ func NewServices( var ( factory ocr3types.MercuryPluginFactory factoryServices []job.ServiceCtx + fErr error ) fCfg := factoryCfg{ orm: orm, @@ -127,31 +128,31 @@ func NewServices( } switch feedID.Version() { case 1: - factory, factoryServices, err = newv1factory(fCfg) - if err != nil { + factory, factoryServices, fErr = newv1factory(fCfg) + if fErr != nil { abort() - return nil, fmt.Errorf("failed to create mercury v1 factory: %w", err) + return nil, fmt.Errorf("failed to create mercury v1 factory: %w", fErr) } srvs = append(srvs, factoryServices...) case 2: - factory, factoryServices, err = newv2factory(fCfg) - if err != nil { + factory, factoryServices, fErr = newv2factory(fCfg) + if fErr != nil { abort() - return nil, fmt.Errorf("failed to create mercury v2 factory: %w", err) + return nil, fmt.Errorf("failed to create mercury v2 factory: %w", fErr) } srvs = append(srvs, factoryServices...) case 3: - factory, factoryServices, err = newv3factory(fCfg) - if err != nil { + factory, factoryServices, fErr = newv3factory(fCfg) + if fErr != nil { abort() - return nil, fmt.Errorf("failed to create mercury v3 factory: %w", err) + return nil, fmt.Errorf("failed to create mercury v3 factory: %w", fErr) } srvs = append(srvs, factoryServices...) case 4: - factory, factoryServices, err = newv4factory(fCfg) - if err != nil { + factory, factoryServices, fErr = newv4factory(fCfg) + if fErr != nil { abort() - return nil, fmt.Errorf("failed to create mercury v4 factory: %w", err) + return nil, fmt.Errorf("failed to create mercury v4 factory: %w", fErr) } srvs = append(srvs, factoryServices...) default: @@ -214,13 +215,14 @@ func newv4factory(factoryCfg factoryCfg) (ocr3types.MercuryPluginFactory, []job. loopEnabled := loopCmd != "" if loopEnabled { - cmdFn, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) + cmdFn, unregisterer, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) if err != nil { return nil, nil, fmt.Errorf("failed to init loop for feed %s: %w", factoryCfg.feedID, err) } // in loop mode, the factory is grpc server, and we need to handle the server lifecycle + // and unregistration of the loop factoryServer := loop.NewMercuryV4Service(mercuryLggr, opts, cmdFn, factoryCfg.ocr2Provider, ds) - srvs = append(srvs, factoryServer) + srvs = append(srvs, factoryServer, unregisterer) // adapt the grpc server to the vanilla mercury plugin factory interface used by the oracle factory = factoryServer } else { @@ -253,13 +255,14 @@ func newv3factory(factoryCfg factoryCfg) (ocr3types.MercuryPluginFactory, []job. loopEnabled := loopCmd != "" if loopEnabled { - cmdFn, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) + cmdFn, unregisterer, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) if err != nil { return nil, nil, fmt.Errorf("failed to init loop for feed %s: %w", factoryCfg.feedID, err) } // in loopp mode, the factory is grpc server, and we need to handle the server lifecycle + // and unregistration of the loop factoryServer := loop.NewMercuryV3Service(mercuryLggr, opts, cmdFn, factoryCfg.ocr2Provider, ds) - srvs = append(srvs, factoryServer) + srvs = append(srvs, factoryServer, unregisterer) // adapt the grpc server to the vanilla mercury plugin factory interface used by the oracle factory = factoryServer } else { @@ -292,13 +295,14 @@ func newv2factory(factoryCfg factoryCfg) (ocr3types.MercuryPluginFactory, []job. loopEnabled := loopCmd != "" if loopEnabled { - cmdFn, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) + cmdFn, unregisterer, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) if err != nil { return nil, nil, fmt.Errorf("failed to init loop for feed %s: %w", factoryCfg.feedID, err) } // in loopp mode, the factory is grpc server, and we need to handle the server lifecycle + // and unregistration of the loop factoryServer := loop.NewMercuryV2Service(mercuryLggr, opts, cmdFn, factoryCfg.ocr2Provider, ds) - srvs = append(srvs, factoryServer) + srvs = append(srvs, factoryServer, unregisterer) // adapt the grpc server to the vanilla mercury plugin factory interface used by the oracle factory = factoryServer } else { @@ -329,13 +333,14 @@ func newv1factory(factoryCfg factoryCfg) (ocr3types.MercuryPluginFactory, []job. loopEnabled := loopCmd != "" if loopEnabled { - cmdFn, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) + cmdFn, unregisterer, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) if err != nil { return nil, nil, fmt.Errorf("failed to init loop for feed %s: %w", factoryCfg.feedID, err) } // in loopp mode, the factory is grpc server, and we need to handle the server lifecycle + // and unregistration of the loop factoryServer := loop.NewMercuryV1Service(mercuryLggr, opts, cmdFn, factoryCfg.ocr2Provider, ds) - srvs = append(srvs, factoryServer) + srvs = append(srvs, factoryServer, unregisterer) // adapt the grpc server to the vanilla mercury plugin factory interface used by the oracle factory = factoryServer } else { @@ -344,20 +349,46 @@ func newv1factory(factoryCfg factoryCfg) (ocr3types.MercuryPluginFactory, []job. return factory, srvs, nil } -func initLoop(cmd string, cfg Config, feedID utils.FeedID, lggr logger.Logger) (func() *exec.Cmd, loop.GRPCOpts, logger.Logger, error) { +func initLoop(cmd string, cfg Config, feedID utils.FeedID, lggr logger.Logger) (func() *exec.Cmd, *loopUnregisterCloser, loop.GRPCOpts, logger.Logger, error) { lggr.Debugw("Initializing Mercury loop", "command", cmd) mercuryLggr := lggr.Named(fmt.Sprintf("MercuryV%d", feedID.Version())).Named(feedID.String()) envVars, err := plugins.ParseEnvFile(env.MercuryPlugin.Env.Get()) if err != nil { - return nil, loop.GRPCOpts{}, nil, fmt.Errorf("failed to parse mercury env file: %w", err) + return nil, nil, loop.GRPCOpts{}, nil, fmt.Errorf("failed to parse mercury env file: %w", err) } + loopID := mercuryLggr.Name() cmdFn, opts, err := cfg.RegisterLOOP(plugins.CmdConfig{ - ID: mercuryLggr.Name(), + ID: loopID, Cmd: cmd, Env: envVars, }) if err != nil { - return nil, loop.GRPCOpts{}, nil, fmt.Errorf("failed to register loop: %w", err) + return nil, nil, loop.GRPCOpts{}, nil, fmt.Errorf("failed to register loop: %w", err) + } + return cmdFn, newLoopUnregister(cfg, loopID), opts, mercuryLggr, nil +} + +// loopUnregisterCloser is a helper to unregister a loop +// as a service +// TODO BCF-3451 all other jobs that use custom plugin providers that should be refactored to use this pattern +// perhaps it can be implemented in the delegate on job delete. +type loopUnregisterCloser struct { + r plugins.RegistrarConfig + id string +} + +func (l *loopUnregisterCloser) Close() error { + l.r.UnregisterLOOP(l.id) + return nil +} + +func (l *loopUnregisterCloser) Start(ctx context.Context) error { + return nil +} + +func newLoopUnregister(r plugins.RegistrarConfig, id string) *loopUnregisterCloser { + return &loopUnregisterCloser{ + r: r, + id: id, } - return cmdFn, opts, mercuryLggr, nil } diff --git a/core/services/ocr2/plugins/mercury/plugin_test.go b/core/services/ocr2/plugins/mercury/plugin_test.go index 22aaf7522de..eb67da53100 100644 --- a/core/services/ocr2/plugins/mercury/plugin_test.go +++ b/core/services/ocr2/plugins/mercury/plugin_test.go @@ -2,6 +2,7 @@ package mercury_test import ( "context" + "errors" "os/exec" "reflect" "testing" @@ -9,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/google/uuid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/logger" @@ -22,6 +24,7 @@ import ( v2 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v2" v3 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v3" v4 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v4" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" mercuryocr2 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/mercury" @@ -92,21 +95,23 @@ var ( // this is kind of gross, but it's the best way to test return values of the services expectedEmbeddedServiceCnt = 3 - expectedLoopServiceCnt = expectedEmbeddedServiceCnt + 1 + expectedLoopServiceCnt = expectedEmbeddedServiceCnt + 2 // factory server and loop unregisterer ) func TestNewServices(t *testing.T) { type args struct { pluginConfig job.JSONConfig feedID utils.FeedID + cfg mercuryocr2.Config } - tests := []struct { + testCases := []struct { name string args args loopMode bool wantLoopFactory any wantServiceCnt int wantErr bool + wantErrStr string }{ { name: "no plugin config error ", @@ -186,6 +191,19 @@ func TestNewServices(t *testing.T) { wantErr: false, wantLoopFactory: &loop.MercuryV3Service{}, }, + { + name: "v3 loop err", + loopMode: true, + args: args{ + pluginConfig: v3jsonCfg, + feedID: v3FeedId, + cfg: mercuryocr2.NewMercuryConfig(1, 1, &testRegistrarConfig{failRegister: true}), + }, + wantServiceCnt: expectedLoopServiceCnt, + wantErr: true, + wantLoopFactory: &loop.MercuryV3Service{}, + wantErrStr: "failed to init loop for feed", + }, { name: "v4 loop", loopMode: true, @@ -198,17 +216,27 @@ func TestNewServices(t *testing.T) { wantLoopFactory: &loop.MercuryV4Service{}, }, } - for _, tt := range tests { + for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { if tt.loopMode { t.Setenv(string(env.MercuryPlugin.Cmd), "fake_cmd") assert.NotEmpty(t, env.MercuryPlugin.Cmd.Get()) } - got, err := newServicesTestWrapper(t, tt.args.pluginConfig, tt.args.feedID) + // use default config if not provided + if tt.args.cfg == nil { + tt.args.cfg = testCfg + } + got, err := newServicesTestWrapper(t, tt.args.pluginConfig, tt.args.feedID, tt.args.cfg) if (err != nil) != tt.wantErr { t.Errorf("NewServices() error = %v, wantErr %v", err, tt.wantErr) return } + if err != nil { + if tt.wantErrStr != "" { + assert.Contains(t, err.Error(), tt.wantErrStr) + } + return + } assert.Len(t, got, tt.wantServiceCnt) if tt.loopMode { foundLoopFactory := false @@ -222,15 +250,97 @@ func TestNewServices(t *testing.T) { } }) } + + t.Run("restartable loop", func(t *testing.T) { + // setup a real loop registry to test restartability + registry := plugins.NewLoopRegistry(logger.TestLogger(t), nil, nil, nil, "") + loopRegistrarConfig := plugins.NewRegistrarConfig(loop.GRPCOpts{}, registry.Register, registry.Unregister) + prodCfg := mercuryocr2.NewMercuryConfig(1, 1, loopRegistrarConfig) + type args struct { + pluginConfig job.JSONConfig + feedID utils.FeedID + cfg mercuryocr2.Config + } + testCases := []struct { + name string + args args + wantErr bool + }{ + { + name: "v1 loop", + args: args{ + pluginConfig: v1jsonCfg, + feedID: v1FeedId, + cfg: prodCfg, + }, + wantErr: false, + }, + { + name: "v2 loop", + args: args{ + pluginConfig: v2jsonCfg, + feedID: v2FeedId, + cfg: prodCfg, + }, + wantErr: false, + }, + { + name: "v3 loop", + args: args{ + pluginConfig: v3jsonCfg, + feedID: v3FeedId, + cfg: prodCfg, + }, + wantErr: false, + }, + { + name: "v4 loop", + args: args{ + pluginConfig: v4jsonCfg, + feedID: v4FeedId, + cfg: prodCfg, + }, + wantErr: false, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(string(env.MercuryPlugin.Cmd), "fake_cmd") + assert.NotEmpty(t, env.MercuryPlugin.Cmd.Get()) + + got, err := newServicesTestWrapper(t, tt.args.pluginConfig, tt.args.feedID, tt.args.cfg) + if (err != nil) != tt.wantErr { + t.Errorf("NewServices() error = %v, wantErr %v", err, tt.wantErr) + return + } + // hack to simulate a restart. we don't have enough boilerplate to start the oracle service + // only care about the subservices so we start all except the oracle, which happens to be the last one + for i := 0; i < len(got)-1; i++ { + require.NoError(t, got[i].Start(tests.Context(t))) + } + // if we don't close the services, we get conflicts with the loop registry + _, err = newServicesTestWrapper(t, tt.args.pluginConfig, tt.args.feedID, tt.args.cfg) + require.ErrorContains(t, err, "plugin already registered") + + // close all services and try again + for i := len(got) - 2; i >= 0; i-- { + require.NoError(t, got[i].Close()) + } + _, err = newServicesTestWrapper(t, tt.args.pluginConfig, tt.args.feedID, tt.args.cfg) + require.NoError(t, err) + }) + } + }) } // we are only varying the version via feedID (and the plugin config) // this wrapper supplies dummy values for the rest of the arguments -func newServicesTestWrapper(t *testing.T, pluginConfig job.JSONConfig, feedID utils.FeedID) ([]job.ServiceCtx, error) { +func newServicesTestWrapper(t *testing.T, pluginConfig job.JSONConfig, feedID utils.FeedID, cfg mercuryocr2.Config) ([]job.ServiceCtx, error) { t.Helper() jb := testJob jb.OCR2OracleSpec.PluginConfig = pluginConfig - return mercuryocr2.NewServices(jb, &testProvider{}, nil, logger.TestLogger(t), testArgsNoPlugin, testCfg, nil, &testDataSourceORM{}, feedID, false) + return mercuryocr2.NewServices(jb, &testProvider{}, nil, logger.TestLogger(t), testArgsNoPlugin, cfg, nil, &testDataSourceORM{}, feedID, false) } type testProvider struct{} @@ -292,16 +402,21 @@ func (*testProvider) ReportCodecV3() v3.ReportCodec { return nil } func (*testProvider) ReportCodecV4() v4.ReportCodec { return nil } // Start implements types.MercuryProvider. -func (*testProvider) Start(context.Context) error { panic("unimplemented") } +func (*testProvider) Start(context.Context) error { return nil } var _ commontypes.MercuryProvider = (*testProvider)(nil) -type testRegistrarConfig struct{} +type testRegistrarConfig struct { + failRegister bool +} func (c *testRegistrarConfig) UnregisterLOOP(ID string) {} // RegisterLOOP implements plugins.RegistrarConfig. -func (*testRegistrarConfig) RegisterLOOP(config plugins.CmdConfig) (func() *exec.Cmd, loop.GRPCOpts, error) { +func (c *testRegistrarConfig) RegisterLOOP(config plugins.CmdConfig) (func() *exec.Cmd, loop.GRPCOpts, error) { + if c.failRegister { + return nil, loop.GRPCOpts{}, errors.New("failed to register") + } return nil, loop.GRPCOpts{}, nil } diff --git a/plugins/registrar.go b/plugins/registrar.go index 2a82f2a6204..8523d3980cc 100644 --- a/plugins/registrar.go +++ b/plugins/registrar.go @@ -6,7 +6,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/loop" ) -// RegistrarConfig generates contains static configuration inher +// RegistrarConfig generates contains static configuration type RegistrarConfig interface { RegisterLOOP(config CmdConfig) (func() *exec.Cmd, loop.GRPCOpts, error) UnregisterLOOP(ID string) From acc38461e5e41790f54bd85a1d1f60c6e4a01ee3 Mon Sep 17 00:00:00 2001 From: Cedric Date: Thu, 12 Dec 2024 18:08:37 +0000 Subject: [PATCH 03/11] [CAPPL-382/CAPPL-366] Miscellaneous fixes (#15666) --- core/scripts/go.mod | 2 +- core/scripts/go.sum | 4 ++-- .../capabilities/workflows/syncer/workflow_syncer_test.go | 4 ++-- core/services/workflows/syncer/handler.go | 2 +- core/services/workflows/syncer/handler_test.go | 8 ++++---- deployment/go.mod | 2 +- deployment/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 ++-- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 ++-- 13 files changed, 22 insertions(+), 22 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 6bab1f30f8e..d29df20e3fa 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -33,7 +33,7 @@ require ( github.com/prometheus/client_golang v1.20.5 github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chainlink-automation v0.8.1 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49 github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 diff --git a/core/scripts/go.sum b/core/scripts/go.sum index f86aad22fb4..c14563ea569 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1142,8 +1142,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0 h1:/1L+v4SxUD2K5RMRbfByyLfePMAgQKeD0onSetPnGmA= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0/go.mod h1:F8xQAIW0ymb2BZhqn89sWZLXreJhM5KDVF6Qb4y44N0= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83 h1:NjrU7KOn3Tk+C6QFo9tQBqeotPKytpBwhn/J1s+yiiY= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49 h1:ZA92CTX9JtEArrxgZw7PNctVxFS+/DmSXumkwf1WiMY= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/core/services/relay/evm/capabilities/workflows/syncer/workflow_syncer_test.go b/core/services/relay/evm/capabilities/workflows/syncer/workflow_syncer_test.go index 066e85e839f..c7c164803cb 100644 --- a/core/services/relay/evm/capabilities/workflows/syncer/workflow_syncer_test.go +++ b/core/services/relay/evm/capabilities/workflows/syncer/workflow_syncer_test.go @@ -418,7 +418,7 @@ func Test_RegistrySyncer_WorkflowRegistered_InitiallyPaused(t *testing.T) { require.NoError(t, err) from := [20]byte(backendTH.ContractsOwner.From) - id, err := workflows.GenerateWorkflowID(from[:], []byte(wantContents), []byte(""), "") + id, err := workflows.GenerateWorkflowID(from[:], "test-wf", []byte(wantContents), []byte(""), "") require.NoError(t, err) giveWorkflow.ID = id @@ -516,7 +516,7 @@ func Test_RegistrySyncer_WorkflowRegistered_InitiallyActivated(t *testing.T) { require.NoError(t, err) from := [20]byte(backendTH.ContractsOwner.From) - id, err := workflows.GenerateWorkflowID(from[:], []byte(wantContents), []byte(""), "") + id, err := workflows.GenerateWorkflowID(from[:], "test-wf", []byte(wantContents), []byte(""), "") require.NoError(t, err) giveWorkflow.ID = id diff --git a/core/services/workflows/syncer/handler.go b/core/services/workflows/syncer/handler.go index f3392a8489a..4ef7f952249 100644 --- a/core/services/workflows/syncer/handler.go +++ b/core/services/workflows/syncer/handler.go @@ -428,7 +428,7 @@ func (h *eventHandler) workflowRegisteredEvent( } // Calculate the hash of the binary and config files - hash, err := pkgworkflows.GenerateWorkflowID(payload.WorkflowOwner, decodedBinary, config, payload.SecretsURL) + hash, err := pkgworkflows.GenerateWorkflowID(payload.WorkflowOwner, payload.WorkflowName, decodedBinary, config, payload.SecretsURL) if err != nil { return fmt.Errorf("failed to generate workflow id: %w", err) } diff --git a/core/services/workflows/syncer/handler_test.go b/core/services/workflows/syncer/handler_test.go index eb8b338158f..f205cbde1cd 100644 --- a/core/services/workflows/syncer/handler_test.go +++ b/core/services/workflows/syncer/handler_test.go @@ -444,7 +444,7 @@ func testRunningWorkflow(t *testing.T, tc testCase) { fetcher = tc.fetcher ) - giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, binary, config, secretsURL) + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "workflow-name", binary, config, secretsURL) require.NoError(t, err) wfID := hex.EncodeToString(giveWFID[:]) @@ -492,7 +492,7 @@ func Test_workflowDeletedHandler(t *testing.T) { }) ) - giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, binary, config, secretsURL) + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "workflow-name", binary, config, secretsURL) require.NoError(t, err) wfIDs := hex.EncodeToString(giveWFID[:]) @@ -584,9 +584,9 @@ func Test_workflowPausedActivatedUpdatedHandler(t *testing.T) { }) ) - giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, binary, config, secretsURL) + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "workflow-name", binary, config, secretsURL) require.NoError(t, err) - updatedWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, binary, updateConfig, secretsURL) + updatedWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "workflow-name", binary, updateConfig, secretsURL) require.NoError(t, err) require.NoError(t, err) diff --git a/deployment/go.mod b/deployment/go.mod index 8c30d54bdff..fc3d70c3900 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -29,7 +29,7 @@ require ( github.com/smartcontractkit/ccip-owner-contracts v0.0.0-20240926212305-a6deabdfce86 github.com/smartcontractkit/chain-selectors v1.0.34 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49 github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.13 github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 diff --git a/deployment/go.sum b/deployment/go.sum index b1ce805ba28..f9fb767d5da 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1411,8 +1411,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0 h1:/1L+v4SxUD2K5RMRbfByyLfePMAgQKeD0onSetPnGmA= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0/go.mod h1:F8xQAIW0ymb2BZhqn89sWZLXreJhM5KDVF6Qb4y44N0= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83 h1:NjrU7KOn3Tk+C6QFo9tQBqeotPKytpBwhn/J1s+yiiY= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49 h1:ZA92CTX9JtEArrxgZw7PNctVxFS+/DmSXumkwf1WiMY= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/go.mod b/go.mod index 2149898f15b..0f8a161768f 100644 --- a/go.mod +++ b/go.mod @@ -79,7 +79,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.34 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49 github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db github.com/smartcontractkit/chainlink-feeds v0.1.1 diff --git a/go.sum b/go.sum index 45a2dfab4fe..76127a91b4b 100644 --- a/go.sum +++ b/go.sum @@ -1125,8 +1125,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0 h1:/1L+v4SxUD2K5RMRbfByyLfePMAgQKeD0onSetPnGmA= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0/go.mod h1:F8xQAIW0ymb2BZhqn89sWZLXreJhM5KDVF6Qb4y44N0= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83 h1:NjrU7KOn3Tk+C6QFo9tQBqeotPKytpBwhn/J1s+yiiY= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49 h1:ZA92CTX9JtEArrxgZw7PNctVxFS+/DmSXumkwf1WiMY= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index c1b012e3641..b87192af47e 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -47,7 +47,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.34 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49 github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.19 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index fb3d895d130..75f4e862f61 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1432,8 +1432,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0 h1:/1L+v4SxUD2K5RMRbfByyLfePMAgQKeD0onSetPnGmA= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0/go.mod h1:F8xQAIW0ymb2BZhqn89sWZLXreJhM5KDVF6Qb4y44N0= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83 h1:NjrU7KOn3Tk+C6QFo9tQBqeotPKytpBwhn/J1s+yiiY= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49 h1:ZA92CTX9JtEArrxgZw7PNctVxFS+/DmSXumkwf1WiMY= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 5f49519cb4b..3d240cccc9e 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -27,7 +27,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.33.0 github.com/slack-go/slack v0.15.0 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.19 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.9 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index cda5cebf370..96861fdc048 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1423,8 +1423,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0 h1:/1L+v4SxUD2K5RMRbfByyLfePMAgQKeD0onSetPnGmA= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241211150100-7683331f64a0/go.mod h1:F8xQAIW0ymb2BZhqn89sWZLXreJhM5KDVF6Qb4y44N0= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83 h1:NjrU7KOn3Tk+C6QFo9tQBqeotPKytpBwhn/J1s+yiiY= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241210192653-a9c706f99e83/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49 h1:ZA92CTX9JtEArrxgZw7PNctVxFS+/DmSXumkwf1WiMY= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241212163958-6a43e61b9d49/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= From 52f364a6cd842fe63c4cab6182f4c9fbbf7d134e Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 12 Dec 2024 13:30:33 -0500 Subject: [PATCH 04/11] Classify Arbitrum rpc server errors (#15488) * Add service errors * Add default service errors * Fix tests * Update giant-eels-jump.md * Update errors.go --- .changeset/giant-eels-jump.md | 5 +++++ core/chains/evm/client/errors.go | 15 +++++++++++++-- core/chains/evm/client/errors_test.go | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 .changeset/giant-eels-jump.md diff --git a/.changeset/giant-eels-jump.md b/.changeset/giant-eels-jump.md new file mode 100644 index 00000000000..5ab8ca875ca --- /dev/null +++ b/.changeset/giant-eels-jump.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +Add error handling for Arbitrum RPC server timeouts. #added diff --git a/core/chains/evm/client/errors.go b/core/chains/evm/client/errors.go index 1075dc40606..bde97185580 100644 --- a/core/chains/evm/client/errors.go +++ b/core/chains/evm/client/errors.go @@ -64,6 +64,7 @@ const ( ServiceUnavailable TerminallyStuck TooManyResults + ServiceTimeout ) type ClientErrors map[int]*regexp.Regexp @@ -160,7 +161,8 @@ var arbitrum = ClientErrors{ Fatal: arbitrumFatal, L2FeeTooLow: regexp.MustCompile(`(: |^)max fee per gas less than block base fee(:|$)`), L2Full: regexp.MustCompile(`(: |^)(queue full|sequencer pending tx pool full, please try again)(:|$)`), - ServiceUnavailable: regexp.MustCompile(`(: |^)502 Bad Gateway: [\s\S]*$|network is unreachable|i/o timeout`), + ServiceUnavailable: regexp.MustCompile(`(: |^)502 Bad Gateway: [\s\S]*$|network is unreachable|i/o timeout|(: |^)503 Service Temporarily Unavailable(:|$)`), + ServiceTimeout: regexp.MustCompile(`(: |^)408 Request Timeout(:|$)`), } // Treasure @@ -398,6 +400,11 @@ func (s *SendError) IsServiceUnavailable(configErrors *ClientErrors) bool { return s.is(ServiceUnavailable, configErrors) || pkgerrors.Is(s.err, commonclient.ErroringNodeError) } +// IsServiceTimeout indicates if the error was caused by a service timeout +func (s *SendError) IsServiceTimeout(configErrors *ClientErrors) bool { + return s.is(ServiceTimeout, configErrors) +} + // IsTerminallyStuck indicates if a transaction was stuck without any chance of inclusion func (s *SendError) IsTerminallyStuckConfigError(configErrors *ClientErrors) bool { return s.is(TerminallyStuck, configErrors) @@ -619,6 +626,10 @@ func ClassifySendError(err error, clientErrors config.ClientErrors, lggr logger. lggr.Errorw(fmt.Sprintf("service unavailable while sending transaction %x", tx.Hash()), "err", sendError, "etx", tx) return commonclient.Retryable } + if sendError.IsServiceTimeout(configErrors) { + lggr.Errorw(fmt.Sprintf("service timed out while sending transaction %x", tx.Hash()), "err", sendError, "etx", tx) + return commonclient.Retryable + } if sendError.IsTimeout() { lggr.Errorw(fmt.Sprintf("timeout while sending transaction %x", tx.Hash()), "err", sendError, "etx", tx) return commonclient.Retryable @@ -666,7 +677,7 @@ var drpc = ClientErrors{ // Linkpool, Blockdaemon, and Chainstack all return "request timed out" if the log results are too large for them to process var defaultClient = ClientErrors{ - TooManyResults: regexp.MustCompile(`request timed out`), + TooManyResults: regexp.MustCompile(`request timed out|408 Request Timed Out`), } // JSON-RPC error codes which can indicate a refusal of the server to process an eth_getLogs request because the result set is too large diff --git a/core/chains/evm/client/errors_test.go b/core/chains/evm/client/errors_test.go index 75ac21597d8..1f9aaa53365 100644 --- a/core/chains/evm/client/errors_test.go +++ b/core/chains/evm/client/errors_test.go @@ -245,6 +245,7 @@ func Test_Eth_Errors(t *testing.T) { {"network is unreachable", true, "Arbitrum"}, {"client error service unavailable", true, "tomlConfig"}, {"[Request ID: 825608a8-fd8a-4b5b-aea7-92999509306d] Error invoking RPC: [Request ID: 825608a8-fd8a-4b5b-aea7-92999509306d] Transaction execution returns a null value for transaction", true, "hedera"}, + {"call failed: 503 Service Temporarily Unavailable: \r\n503 Service Temporarily Unavailable\r\n\r\n

503 Service Temporarily Unavailable

\r\n\r\n\r\n", true, "Arbitrum"}, } for _, test := range tests { err = evmclient.NewSendErrorS(test.message) @@ -260,6 +261,20 @@ func Test_Eth_Errors(t *testing.T) { } }) + t.Run("IsServiceTimeout", func(t *testing.T) { + tests := []errorCase{ + {"call failed: 408 Request Timeout: {", true, "Arbitrum"}, + {"408 Request Timeout: {\"id\":303,\"jsonrpc\":\"2.0\",\"error\":{\"code\\\":-32009,\\\"message\\\":\\\"request timeout\\\"}}\",\"errVerbose\":\"408 Request Timeout:\n", true, "Arbitrum"}, + {"request timeout", false, "tomlConfig"}, + } + for _, test := range tests { + err = evmclient.NewSendErrorS(test.message) + assert.Equal(t, err.IsServiceTimeout(clientErrors), test.expect) + err = newSendErrorWrapped(test.message) + assert.Equal(t, err.IsServiceTimeout(clientErrors), test.expect) + } + }) + t.Run("IsTxFeeExceedsCap", func(t *testing.T) { tests := []errorCase{ {"tx fee (1.10 ether) exceeds the configured cap (1.00 ether)", true, "geth"}, From f6e3f68ff468fd7f6ea1093743641c3b805f8832 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 12 Dec 2024 11:48:44 -0800 Subject: [PATCH 05/11] chore: update go-conditional-tests to 0.2.0 (#15652) --- .github/workflows/ci-core-partial.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-core-partial.yml b/.github/workflows/ci-core-partial.yml index c9752d4e1e4..35f689090e8 100644 --- a/.github/workflows/ci-core-partial.yml +++ b/.github/workflows/ci-core-partial.yml @@ -46,6 +46,7 @@ jobs: permissions: id-token: write contents: write + actions: write strategy: fail-fast: false matrix: @@ -86,7 +87,7 @@ jobs: go-mod-download-directory: ${{ matrix.type.test-suite == 'ccip-deployment' && matrix.type.module-directory || '' }} - name: Build Tests - uses: smartcontractkit/.github/apps/go-conditional-tests@37882e110590e636627a26371bdbd56ddfcce821 # go-conditional-tests@0.1.0 + uses: smartcontractkit/.github/apps/go-conditional-tests@57f99fbea73056c490c766d50ef582a13ec4f3bb # go-conditional-tests@0.2.0 timeout-minutes: 10 with: pipeline-step: "build" @@ -98,7 +99,7 @@ jobs: build-flags: ${{ matrix.type.build-flags }} - name: Run Tests - uses: smartcontractkit/.github/apps/go-conditional-tests@37882e110590e636627a26371bdbd56ddfcce821 # go-conditional-tests@0.1.0 + uses: smartcontractkit/.github/apps/go-conditional-tests@57f99fbea73056c490c766d50ef582a13ec4f3bb # go-conditional-tests@0.2.0 timeout-minutes: 15 env: CL_DATABASE_URL: ${{ env.DB_URL }} @@ -112,7 +113,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Update Test Index - uses: smartcontractkit/.github/apps/go-conditional-tests@37882e110590e636627a26371bdbd56ddfcce821 # go-conditional-tests@0.1.0 + uses: smartcontractkit/.github/apps/go-conditional-tests@57f99fbea73056c490c766d50ef582a13ec4f3bb # go-conditional-tests@0.2.0 with: pipeline-step: "update" collect-coverage: ${{ needs.filter.outputs.should-collect-coverage }} @@ -130,7 +131,7 @@ jobs: if: ${{ needs.filter.outputs.should-collect-coverage == 'true' }} runs-on: ubuntu-latest steps: - - name: Checkout the repo + - name: Checkout the repo uses: actions/checkout@v4.2.1 with: # fetches all history for all tags and branches to provide more metadata for sonar reports From 52c2db4bff51a5aa2cd67e1c3af71023f1e295de Mon Sep 17 00:00:00 2001 From: "Simon B.Robert" Date: Thu, 12 Dec 2024 15:25:12 -0500 Subject: [PATCH 06/11] Support RMN for CCIP v2 view (#15611) * Support RMN for CCIP v2 view * Write hex strings instead of bytes * Add contract address in error message --- deployment/ccip/changeset/state.go | 9 ++ deployment/ccip/view/v1_6/rmnhome.go | 214 +++++++++++++++++++++++++++ deployment/ccip/view/view.go | 2 + 3 files changed, 225 insertions(+) create mode 100644 deployment/ccip/view/v1_6/rmnhome.go diff --git a/deployment/ccip/changeset/state.go b/deployment/ccip/changeset/state.go index af982f35e0a..45ea9e8f5b8 100644 --- a/deployment/ccip/changeset/state.go +++ b/deployment/ccip/changeset/state.go @@ -161,6 +161,15 @@ func (c CCIPChainState) GenerateView() (view.ChainView, error) { } chainView.RMN[c.RMNRemote.Address().Hex()] = rmnView } + + if c.RMNHome != nil { + rmnHomeView, err := v1_6.GenerateRMNHomeView(c.RMNHome) + if err != nil { + return chainView, errors.Wrapf(err, "failed to generate rmn home view for rmn home %s", c.RMNHome.Address().String()) + } + chainView.RMNHome[c.RMNHome.Address().Hex()] = rmnHomeView + } + if c.FeeQuoter != nil && c.Router != nil && c.TokenAdminRegistry != nil { fqView, err := v1_6.GenerateFeeQuoterView(c.FeeQuoter, c.Router, c.TokenAdminRegistry) if err != nil { diff --git a/deployment/ccip/view/v1_6/rmnhome.go b/deployment/ccip/view/v1_6/rmnhome.go new file mode 100644 index 00000000000..82d39074d6f --- /dev/null +++ b/deployment/ccip/view/v1_6/rmnhome.go @@ -0,0 +1,214 @@ +package v1_6 + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/smartcontractkit/chainlink/deployment/common/view/types" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/rmn_home" +) + +type RMNHomeView struct { + types.ContractMetaData + CandidateConfig *RMNHomeVersionedConfig `json:"candidateConfig,omitempty"` + ActiveConfig *RMNHomeVersionedConfig `json:"activeConfig,omitempty"` +} + +type RMNHomeVersionedConfig struct { + Version uint32 `json:"version"` + StaticConfig RMNHomeStaticConfig `json:"staticConfig"` + DynamicConfig RMNHomeDynamicConfig `json:"dynamicConfig"` + Digest [32]byte `json:"digest"` +} + +func decodeHexString(hexStr string, expectedLength int) ([]byte, error) { + bytes, err := hex.DecodeString(hexStr) + if err != nil { + return nil, err + } + if len(bytes) != expectedLength { + return nil, fmt.Errorf("invalid length: expected %d, got %d", expectedLength, len(bytes)) + } + return bytes, nil +} + +func (c RMNHomeVersionedConfig) MarshalJSON() ([]byte, error) { + type Alias RMNHomeVersionedConfig + return json.Marshal(&struct { + Digest string `json:"digest"` + *Alias + }{ + Digest: hex.EncodeToString(c.Digest[:]), + Alias: (*Alias)(&c), + }) +} + +func (c *RMNHomeVersionedConfig) UnmarshalJSON(data []byte) error { + type Alias RMNHomeVersionedConfig + aux := &struct { + Digest string `json:"digest"` + *Alias + }{ + Alias: (*Alias)(c), + } + + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + digestBytes, err := decodeHexString(aux.Digest, 32) + if err != nil { + return err + } + copy(c.Digest[:], digestBytes) + return nil +} + +type RMNHomeStaticConfig struct { + Nodes []RMNHomeNode `json:"nodes"` +} + +type RMNHomeDynamicConfig struct { + SourceChains []RMNHomeSourceChain `json:"sourceChains"` +} + +type RMNHomeSourceChain struct { + ChainSelector uint64 `json:"selector"` + F uint64 `json:"f"` + ObserverNodesBitmap *big.Int `json:"observerNodesBitmap"` +} + +type RMNHomeNode struct { + PeerId [32]byte `json:"peerId"` + OffchainPublicKey [32]byte `json:"offchainPublicKey"` +} + +func (n RMNHomeNode) MarshalJSON() ([]byte, error) { + type Alias RMNHomeNode + return json.Marshal(&struct { + PeerId string `json:"peerId"` + OffchainPublicKey string `json:"offchainPublicKey"` + *Alias + }{ + PeerId: hex.EncodeToString(n.PeerId[:]), + OffchainPublicKey: hex.EncodeToString(n.OffchainPublicKey[:]), + Alias: (*Alias)(&n), + }) +} + +func (n *RMNHomeNode) UnmarshalJSON(data []byte) error { + type Alias RMNHomeNode + aux := &struct { + PeerId string `json:"peerId"` + OffchainPublicKey string `json:"offchainPublicKey"` + *Alias + }{ + Alias: (*Alias)(n), + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + peerIdBytes, err := decodeHexString(aux.PeerId, 32) + if err != nil { + return err + } + copy(n.PeerId[:], peerIdBytes) + + offchainPublicKeyBytes, err := decodeHexString(aux.OffchainPublicKey, 32) + if err != nil { + return err + } + copy(n.OffchainPublicKey[:], offchainPublicKeyBytes) + + return nil +} + +type DigestFunc func(*bind.CallOpts) ([32]byte, error) + +func mapNodes(nodes []rmn_home.RMNHomeNode) []RMNHomeNode { + result := make([]RMNHomeNode, len(nodes)) + for i, node := range nodes { + result[i] = RMNHomeNode{ + PeerId: node.PeerId, + OffchainPublicKey: node.OffchainPublicKey, + } + } + return result +} + +func mapSourceChains(chains []rmn_home.RMNHomeSourceChain) []RMNHomeSourceChain { + result := make([]RMNHomeSourceChain, len(chains)) + for i, chain := range chains { + result[i] = RMNHomeSourceChain{ + ChainSelector: chain.ChainSelector, + F: chain.F, + ObserverNodesBitmap: chain.ObserverNodesBitmap, + } + } + return result +} + +func generateRmnHomeVersionedConfig(reader *rmn_home.RMNHome, digestFunc DigestFunc) (*RMNHomeVersionedConfig, error) { + address := reader.Address() + digest, err := digestFunc(nil) + if err != nil { + return nil, fmt.Errorf("failed to get digest for contract %s: %w", address, err) + } + + if digest == [32]byte{} { + return nil, nil + } + + config, err := reader.GetConfig(nil, digest) + if err != nil { + return nil, fmt.Errorf("failed to get config for contract %s: %w", address, err) + } + + staticConfig := RMNHomeStaticConfig{ + Nodes: mapNodes(config.VersionedConfig.StaticConfig.Nodes), + } + + dynamicConfig := RMNHomeDynamicConfig{ + SourceChains: mapSourceChains(config.VersionedConfig.DynamicConfig.SourceChains), + } + + return &RMNHomeVersionedConfig{ + Version: config.VersionedConfig.Version, + Digest: config.VersionedConfig.ConfigDigest, + StaticConfig: staticConfig, + DynamicConfig: dynamicConfig, + }, nil +} + +func GenerateRMNHomeView(rmnReader *rmn_home.RMNHome) (RMNHomeView, error) { + if rmnReader == nil { + return RMNHomeView{}, nil + } + + address := rmnReader.Address() + + activeConfig, err := generateRmnHomeVersionedConfig(rmnReader, rmnReader.GetActiveDigest) + if err != nil { + return RMNHomeView{}, fmt.Errorf("failed to generate active config for contract %s: %w", address, err) + } + + candidateConfig, err := generateRmnHomeVersionedConfig(rmnReader, rmnReader.GetCandidateDigest) + if err != nil { + return RMNHomeView{}, fmt.Errorf("failed to generate candidate config for contract %s: %w", address, err) + } + + contractMetaData, err := types.NewContractMetaData(rmnReader, rmnReader.Address()) + if err != nil { + return RMNHomeView{}, fmt.Errorf("failed to create contract metadata for contract %s: %w", address, err) + } + + return RMNHomeView{ + ContractMetaData: contractMetaData, + CandidateConfig: candidateConfig, + ActiveConfig: activeConfig, + }, nil +} diff --git a/deployment/ccip/view/view.go b/deployment/ccip/view/view.go index 77781a8a31a..4f216d13008 100644 --- a/deployment/ccip/view/view.go +++ b/deployment/ccip/view/view.go @@ -22,6 +22,7 @@ type ChainView struct { // v1.6 FeeQuoter map[string]v1_6.FeeQuoterView `json:"feeQuoter,omitempty"` NonceManager map[string]v1_6.NonceManagerView `json:"nonceManager,omitempty"` + RMNHome map[string]v1_6.RMNHomeView `json:"rmnHome,omitempty"` RMN map[string]v1_6.RMNRemoteView `json:"rmn,omitempty"` OnRamp map[string]v1_6.OnRampView `json:"onRamp,omitempty"` OffRamp map[string]v1_6.OffRampView `json:"offRamp,omitempty"` @@ -46,6 +47,7 @@ func NewChain() ChainView { // v1.6 FeeQuoter: make(map[string]v1_6.FeeQuoterView), NonceManager: make(map[string]v1_6.NonceManagerView), + RMNHome: make(map[string]v1_6.RMNHomeView), RMN: make(map[string]v1_6.RMNRemoteView), OnRamp: make(map[string]v1_6.OnRampView), OffRamp: make(map[string]v1_6.OffRampView), From 7a5e54c2c73ca329350de3a43e7d218658da7cd4 Mon Sep 17 00:00:00 2001 From: Makram Date: Fri, 13 Dec 2024 01:54:30 +0200 Subject: [PATCH 07/11] deployment/ccip/changeset: mcms optional ccip home cses (#15658) * deployment/ccip/changeset: mcms optional ccip home cses Add MCMS optionality for the rest of the CCIPHome changesets, and organize things a little bit better by moving the AddDON changeset into cs_ccip_home.go out of cs_add_chain.go * fixes * pr comments * one more comment * fix logger --- .../ccip/changeset/accept_ownership_test.go | 4 +- deployment/ccip/changeset/cs_add_chain.go | 170 +-------- .../ccip/changeset/cs_add_chain_test.go | 43 ++- deployment/ccip/changeset/cs_ccip_home.go | 332 ++++++++++++++++-- .../ccip/changeset/cs_ccip_home_test.go | 185 ++++++++-- .../ccip/changeset/cs_initial_add_chain.go | 2 +- 6 files changed, 489 insertions(+), 247 deletions(-) diff --git a/deployment/ccip/changeset/accept_ownership_test.go b/deployment/ccip/changeset/accept_ownership_test.go index 1dbef8e7a0b..9b71e0ad5cb 100644 --- a/deployment/ccip/changeset/accept_ownership_test.go +++ b/deployment/ccip/changeset/accept_ownership_test.go @@ -23,11 +23,11 @@ func Test_NewAcceptOwnershipChangeset(t *testing.T) { dest := allChains[1] timelockContracts := map[uint64]*proposalutils.TimelockExecutionContracts{ - source: &proposalutils.TimelockExecutionContracts{ + source: { Timelock: state.Chains[source].Timelock, CallProxy: state.Chains[source].CallProxy, }, - dest: &proposalutils.TimelockExecutionContracts{ + dest: { Timelock: state.Chains[dest].Timelock, CallProxy: state.Chains[dest].CallProxy, }, diff --git a/deployment/ccip/changeset/cs_add_chain.go b/deployment/ccip/changeset/cs_add_chain.go index b3d0df04c93..ddb6e61d5ba 100644 --- a/deployment/ccip/changeset/cs_add_chain.go +++ b/deployment/ccip/changeset/cs_add_chain.go @@ -8,18 +8,14 @@ import ( "github.com/smartcontractkit/chainlink-ccip/chainconfig" "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" - - "github.com/smartcontractkit/chainlink/deployment/ccip/changeset/internal" "github.com/smartcontractkit/chainlink/deployment/common/proposalutils" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_home" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry" "github.com/smartcontractkit/ccip-owner-contracts/pkg/gethwrappers" "github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/mcms" "github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock" "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_home" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/fee_quoter" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/onramp" ) @@ -136,135 +132,6 @@ func NewChainInboundChangeset( }, nil } -type AddDonAndSetCandidateChangesetConfig struct { - HomeChainSelector uint64 - FeedChainSelector uint64 - NewChainSelector uint64 - PluginType types.PluginType - NodeIDs []string - CCIPOCRParams CCIPOCRParams -} - -func (a AddDonAndSetCandidateChangesetConfig) Validate(e deployment.Environment, state CCIPOnChainState) (deployment.Nodes, error) { - if a.HomeChainSelector == 0 { - return nil, fmt.Errorf("HomeChainSelector must be set") - } - if a.FeedChainSelector == 0 { - return nil, fmt.Errorf("FeedChainSelector must be set") - } - if a.NewChainSelector == 0 { - return nil, fmt.Errorf("ocr config chain selector must be set") - } - if a.PluginType != types.PluginTypeCCIPCommit && - a.PluginType != types.PluginTypeCCIPExec { - return nil, fmt.Errorf("PluginType must be set to either CCIPCommit or CCIPExec") - } - // TODO: validate token config - if len(a.NodeIDs) == 0 { - return nil, fmt.Errorf("nodeIDs must be set") - } - nodes, err := deployment.NodeInfo(a.NodeIDs, e.Offchain) - if err != nil { - return nil, fmt.Errorf("get node info: %w", err) - } - - // check that chain config is set up for the new chain - chainConfig, err := state.Chains[a.HomeChainSelector].CCIPHome.GetChainConfig(nil, a.NewChainSelector) - if err != nil { - return nil, fmt.Errorf("get all chain configs: %w", err) - } - - // FChain should never be zero if a chain config is set in CCIPHome - if chainConfig.FChain == 0 { - return nil, fmt.Errorf("chain config not set up for new chain %d", a.NewChainSelector) - } - - err = a.CCIPOCRParams.Validate() - if err != nil { - return nil, fmt.Errorf("invalid ccip ocr params: %w", err) - } - - if e.OCRSecrets.IsEmpty() { - return nil, fmt.Errorf("OCR secrets must be set") - } - - return nodes, nil -} - -// AddDonAndSetCandidateChangeset adds new DON for destination to home chain -// and sets the commit plugin config as candidateConfig for the don. -func AddDonAndSetCandidateChangeset( - e deployment.Environment, - cfg AddDonAndSetCandidateChangesetConfig, -) (deployment.ChangesetOutput, error) { - state, err := LoadOnchainState(e) - if err != nil { - return deployment.ChangesetOutput{}, err - } - - nodes, err := cfg.Validate(e, state) - if err != nil { - return deployment.ChangesetOutput{}, fmt.Errorf("%w: %w", deployment.ErrInvalidConfig, err) - } - - newDONArgs, err := internal.BuildOCR3ConfigForCCIPHome( - e.OCRSecrets, - state.Chains[cfg.NewChainSelector].OffRamp, - e.Chains[cfg.NewChainSelector], - nodes.NonBootstraps(), - state.Chains[cfg.HomeChainSelector].RMNHome.Address(), - cfg.CCIPOCRParams.OCRParameters, - cfg.CCIPOCRParams.CommitOffChainConfig, - cfg.CCIPOCRParams.ExecuteOffChainConfig, - ) - if err != nil { - return deployment.ChangesetOutput{}, err - } - latestDon, err := internal.LatestCCIPDON(state.Chains[cfg.HomeChainSelector].CapabilityRegistry) - if err != nil { - return deployment.ChangesetOutput{}, err - } - commitConfig, ok := newDONArgs[cfg.PluginType] - if !ok { - return deployment.ChangesetOutput{}, fmt.Errorf("missing commit plugin in ocr3Configs") - } - donID := latestDon.Id + 1 - addDonOp, err := newDonWithCandidateOp( - donID, commitConfig, - state.Chains[cfg.HomeChainSelector].CapabilityRegistry, - nodes.NonBootstraps(), - ) - if err != nil { - return deployment.ChangesetOutput{}, err - } - - var ( - timelocksPerChain = map[uint64]common.Address{ - cfg.HomeChainSelector: state.Chains[cfg.HomeChainSelector].Timelock.Address(), - } - proposerMCMSes = map[uint64]*gethwrappers.ManyChainMultiSig{ - cfg.HomeChainSelector: state.Chains[cfg.HomeChainSelector].ProposerMcm, - } - ) - prop, err := proposalutils.BuildProposalFromBatches( - timelocksPerChain, - proposerMCMSes, - []timelock.BatchChainOperation{{ - ChainIdentifier: mcms.ChainIdentifier(cfg.HomeChainSelector), - Batch: []mcms.Operation{addDonOp}, - }}, - "setCandidate for commit and AddDon on new Chain", - 0, // minDelay - ) - if err != nil { - return deployment.ChangesetOutput{}, fmt.Errorf("failed to build proposal from batch: %w", err) - } - - return deployment.ChangesetOutput{ - Proposals: []timelock.MCMSWithTimelockProposal{*prop}, - }, nil -} - func applyChainConfigUpdatesOp( e deployment.Environment, state CCIPOnChainState, @@ -304,38 +171,3 @@ func applyChainConfigUpdatesOp( Value: big.NewInt(0), }, nil } - -// newDonWithCandidateOp sets the candidate commit config by calling setCandidate on CCIPHome contract through the AddDON call on CapReg contract -// This should be done first before calling any other UpdateDON calls -// This proposes to set up OCR3 config for the commit plugin for the DON -func newDonWithCandidateOp( - donID uint32, - pluginConfig ccip_home.CCIPHomeOCR3Config, - capReg *capabilities_registry.CapabilitiesRegistry, - nodes deployment.Nodes, -) (mcms.Operation, error) { - encodedSetCandidateCall, err := internal.CCIPHomeABI.Pack( - "setCandidate", - donID, - pluginConfig.PluginType, - pluginConfig, - [32]byte{}, - ) - if err != nil { - return mcms.Operation{}, fmt.Errorf("pack set candidate call: %w", err) - } - addDonTx, err := capReg.AddDON(deployment.SimTransactOpts(), nodes.PeerIDs(), []capabilities_registry.CapabilitiesRegistryCapabilityConfiguration{ - { - CapabilityId: internal.CCIPCapabilityID, - Config: encodedSetCandidateCall, - }, - }, false, false, nodes.DefaultF()) - if err != nil { - return mcms.Operation{}, fmt.Errorf("could not generate add don tx w/ commit config: %w", err) - } - return mcms.Operation{ - To: capReg.Address(), - Data: addDonTx.Data(), - Value: big.NewInt(0), - }, nil -} diff --git a/deployment/ccip/changeset/cs_add_chain_test.go b/deployment/ccip/changeset/cs_add_chain_test.go index 96b77f1bd7d..a8fdf50b0c1 100644 --- a/deployment/ccip/changeset/cs_add_chain_test.go +++ b/deployment/ccip/changeset/cs_add_chain_test.go @@ -179,16 +179,9 @@ func TestAddChainInbound(t *testing.T) { assertTimelockOwnership(t, e, initialDeploy, state) - nodes, err := deployment.NodeInfo(e.Env.NodeIDs, e.Env.Offchain) - require.NoError(t, err) - // TODO This currently is not working - Able to send the request here but request gets stuck in execution // Send a new message and expect that this is delivered once the chain is completely set up as inbound //TestSendRequest(t, e.Env, state, initialDeploy[0], newChain, true) - var nodeIDs []string - for _, node := range nodes { - nodeIDs = append(nodeIDs, node.NodeID) - } _, err = commonchangeset.ApplyChangesets(t, e.Env, map[uint64]*proposalutils.TimelockExecutionContracts{ e.HomeChainSel: { @@ -203,31 +196,37 @@ func TestAddChainInbound(t *testing.T) { { Changeset: commonchangeset.WrapChangeSet(AddDonAndSetCandidateChangeset), Config: AddDonAndSetCandidateChangesetConfig{ - HomeChainSelector: e.HomeChainSel, - FeedChainSelector: e.FeedChainSel, - NewChainSelector: newChain, - PluginType: types.PluginTypeCCIPCommit, - NodeIDs: nodeIDs, - CCIPOCRParams: DefaultOCRParams( - e.FeedChainSel, - tokenConfig.GetTokenInfo(logger.TestLogger(t), state.Chains[newChain].LinkToken, state.Chains[newChain].Weth9), - nil, - ), + SetCandidateChangesetConfig: SetCandidateChangesetConfig{ + HomeChainSelector: e.HomeChainSel, + FeedChainSelector: e.FeedChainSel, + DONChainSelector: newChain, + PluginType: types.PluginTypeCCIPCommit, + CCIPOCRParams: DefaultOCRParams( + e.FeedChainSel, + tokenConfig.GetTokenInfo(logger.TestLogger(t), state.Chains[newChain].LinkToken, state.Chains[newChain].Weth9), + nil, + ), + MCMS: &MCMSConfig{ + MinDelay: 0, + }, + }, }, }, { - Changeset: commonchangeset.WrapChangeSet(SetCandidatePluginChangeset), - Config: AddDonAndSetCandidateChangesetConfig{ + Changeset: commonchangeset.WrapChangeSet(SetCandidateChangeset), + Config: SetCandidateChangesetConfig{ HomeChainSelector: e.HomeChainSel, FeedChainSelector: e.FeedChainSel, - NewChainSelector: newChain, + DONChainSelector: newChain, PluginType: types.PluginTypeCCIPExec, - NodeIDs: nodeIDs, CCIPOCRParams: DefaultOCRParams( e.FeedChainSel, tokenConfig.GetTokenInfo(logger.TestLogger(t), state.Chains[newChain].LinkToken, state.Chains[newChain].Weth9), nil, ), + MCMS: &MCMSConfig{ + MinDelay: 0, + }, }, }, { @@ -235,13 +234,13 @@ func TestAddChainInbound(t *testing.T) { Config: PromoteAllCandidatesChangesetConfig{ HomeChainSelector: e.HomeChainSel, DONChainSelector: newChain, - NodeIDs: nodeIDs, MCMS: &MCMSConfig{ MinDelay: 0, }, }, }, }) + require.NoError(t, err) // verify if the configs are updated require.NoError(t, ValidateCCIPHomeConfigSetUp( diff --git a/deployment/ccip/changeset/cs_ccip_home.go b/deployment/ccip/changeset/cs_ccip_home.go index 202d4216b60..22fb1fc23fa 100644 --- a/deployment/ccip/changeset/cs_ccip_home.go +++ b/deployment/ccip/changeset/cs_ccip_home.go @@ -10,27 +10,34 @@ import ( "github.com/smartcontractkit/ccip-owner-contracts/pkg/gethwrappers" "github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/mcms" "github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/ccip/changeset/internal" "github.com/smartcontractkit/chainlink/deployment/common/proposalutils" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types" cctypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_home" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry" ) var ( + _ deployment.ChangeSet[AddDonAndSetCandidateChangesetConfig] = AddDonAndSetCandidateChangeset _ deployment.ChangeSet[PromoteAllCandidatesChangesetConfig] = PromoteAllCandidatesChangeset - _ deployment.ChangeSet[AddDonAndSetCandidateChangesetConfig] = SetCandidatePluginChangeset + _ deployment.ChangeSet[SetCandidateChangesetConfig] = SetCandidateChangeset ) type PromoteAllCandidatesChangesetConfig struct { HomeChainSelector uint64 + // DONChainSelector is the chain selector of the DON that we want to promote the candidate config of. // Note that each (chain, ccip capability version) pair has a unique DON ID. DONChainSelector uint64 - NodeIDs []string - MCMS *MCMSConfig + + // MCMS is optional MCMS configuration, if provided the changeset will generate an MCMS proposal. + // If nil, the changeset will execute the commands directly using the deployer key + // of the provided environment. + MCMS *MCMSConfig } func (p PromoteAllCandidatesChangesetConfig) Validate(e deployment.Environment, state CCIPOnChainState) (deployment.Nodes, error) { @@ -40,7 +47,7 @@ func (p PromoteAllCandidatesChangesetConfig) Validate(e deployment.Environment, if err := deployment.IsValidChainSelector(p.DONChainSelector); err != nil { return nil, fmt.Errorf("don chain selector invalid: %w", err) } - if len(p.NodeIDs) == 0 { + if len(e.NodeIDs) == 0 { return nil, fmt.Errorf("NodeIDs must be set") } if state.Chains[p.HomeChainSelector].CCIPHome == nil { @@ -49,8 +56,12 @@ func (p PromoteAllCandidatesChangesetConfig) Validate(e deployment.Environment, if state.Chains[p.HomeChainSelector].CapabilityRegistry == nil { return nil, fmt.Errorf("CapabilityRegistry contract does not exist") } + if state.Chains[p.DONChainSelector].OffRamp == nil { + // should not be possible, but a defensive check. + return nil, fmt.Errorf("OffRamp contract does not exist") + } - nodes, err := deployment.NodeInfo(p.NodeIDs, e.Offchain) + nodes, err := deployment.NodeInfo(e.NodeIDs, e.Offchain) if err != nil { return nil, fmt.Errorf("fetch node info: %w", err) } @@ -96,7 +107,10 @@ func (p PromoteAllCandidatesChangesetConfig) Validate(e deployment.Environment, } // PromoteAllCandidatesChangeset generates a proposal to call promoteCandidate on the CCIPHome through CapReg. -// This needs to be called after SetCandidateProposal is executed. +// Note that a DON must exist prior to being able to use this changeset effectively, +// i.e AddDonAndSetCandidateChangeset must be called first. +// This can also be used to promote a 0x0 candidate config to be the active, effectively shutting down the DON. +// At that point you can call the RemoveDON changeset to remove the DON entirely from the capability registry. func PromoteAllCandidatesChangeset( e deployment.Environment, cfg PromoteAllCandidatesChangesetConfig, @@ -160,8 +174,122 @@ func PromoteAllCandidatesChangeset( }, nil } -// SetCandidatePluginChangeset calls setCandidate on the CCIPHome for setting up OCR3 exec Plugin config for the new chain. -func SetCandidatePluginChangeset( +// AddDonAndSetCandidateChangesetConfig is a separate config struct +// because the validation is slightly different from SetCandidateChangesetConfig. +// In particular, we check to make sure we don't already have a DON for the chain. +type AddDonAndSetCandidateChangesetConfig struct { + SetCandidateChangesetConfig +} + +func (a AddDonAndSetCandidateChangesetConfig) Validate(e deployment.Environment, state CCIPOnChainState) (deployment.Nodes, error) { + nodes, err := a.SetCandidateChangesetConfig.Validate(e, state) + if err != nil { + return nil, err + } + + // check if a DON already exists for this chain + donID, err := internal.DonIDForChain( + state.Chains[a.HomeChainSelector].CapabilityRegistry, + state.Chains[a.HomeChainSelector].CCIPHome, + a.DONChainSelector, + ) + if err != nil { + return nil, fmt.Errorf("fetch don id for chain: %w", err) + } + if donID != 0 { + return nil, fmt.Errorf("don already exists in CR for chain %d, it has id %d", a.DONChainSelector, donID) + } + + return nodes, nil +} + +type SetCandidateChangesetConfig struct { + HomeChainSelector uint64 + FeedChainSelector uint64 + + // DONChainSelector is the chain selector of the chain where the DON will be added. + DONChainSelector uint64 + + PluginType types.PluginType + // Note that the PluginType field is used to determine which field in CCIPOCRParams is used. + CCIPOCRParams CCIPOCRParams + + // MCMS is optional MCMS configuration, if provided the changeset will generate an MCMS proposal. + // If nil, the changeset will execute the commands directly using the deployer key + // of the provided environment. + MCMS *MCMSConfig +} + +func (s SetCandidateChangesetConfig) Validate(e deployment.Environment, state CCIPOnChainState) (deployment.Nodes, error) { + if err := deployment.IsValidChainSelector(s.HomeChainSelector); err != nil { + return nil, fmt.Errorf("home chain selector invalid: %w", err) + } + if err := deployment.IsValidChainSelector(s.FeedChainSelector); err != nil { + return nil, fmt.Errorf("feed chain selector invalid: %w", err) + } + if err := deployment.IsValidChainSelector(s.DONChainSelector); err != nil { + return nil, fmt.Errorf("don chain selector invalid: %w", err) + } + if len(e.NodeIDs) == 0 { + return nil, fmt.Errorf("nodeIDs must be set") + } + if state.Chains[s.HomeChainSelector].CCIPHome == nil { + return nil, fmt.Errorf("CCIPHome contract does not exist") + } + if state.Chains[s.HomeChainSelector].CapabilityRegistry == nil { + return nil, fmt.Errorf("CapabilityRegistry contract does not exist") + } + if state.Chains[s.DONChainSelector].OffRamp == nil { + // should not be possible, but a defensive check. + return nil, fmt.Errorf("OffRamp contract does not exist on don chain selector %d", s.DONChainSelector) + } + if s.PluginType != types.PluginTypeCCIPCommit && + s.PluginType != types.PluginTypeCCIPExec { + return nil, fmt.Errorf("PluginType must be set to either CCIPCommit or CCIPExec") + } + + nodes, err := deployment.NodeInfo(e.NodeIDs, e.Offchain) + if err != nil { + return nil, fmt.Errorf("get node info: %w", err) + } + + // TODO: validate token config + // TODO: validate gas config + + // check that chain config is set up for the new chain + chainConfig, err := state.Chains[s.HomeChainSelector].CCIPHome.GetChainConfig(nil, s.DONChainSelector) + if err != nil { + return nil, fmt.Errorf("get all chain configs: %w", err) + } + + // FChain should never be zero if a chain config is set in CCIPHome + if chainConfig.FChain == 0 { + return nil, fmt.Errorf("chain config not set up for new chain %d", s.DONChainSelector) + } + + err = s.CCIPOCRParams.Validate() + if err != nil { + return nil, fmt.Errorf("invalid ccip ocr params: %w", err) + } + + if e.OCRSecrets.IsEmpty() { + return nil, fmt.Errorf("OCR secrets must be set") + } + + return nodes, nil +} + +// AddDonAndSetCandidateChangeset adds new DON for destination to home chain +// and sets the plugin config as candidateConfig for the don. +// +// This is the first step to creating a CCIP DON and must be executed before any +// other changesets (SetCandidateChangeset, PromoteAllCandidatesChangeset) +// can be executed. +// +// Note that these operations must be done together because the createDON call +// in the capability registry calls the capability config contract, so we must +// provide suitable calldata for CCIPHome. +func AddDonAndSetCandidateChangeset( e deployment.Environment, cfg AddDonAndSetCandidateChangesetConfig, ) (deployment.ChangesetOutput, error) { @@ -175,10 +303,153 @@ func SetCandidatePluginChangeset( return deployment.ChangesetOutput{}, fmt.Errorf("%w: %w", deployment.ErrInvalidConfig, err) } + txOpts := e.Chains[cfg.HomeChainSelector].DeployerKey + if cfg.MCMS != nil { + txOpts = deployment.SimTransactOpts() + } + newDONArgs, err := internal.BuildOCR3ConfigForCCIPHome( e.OCRSecrets, - state.Chains[cfg.NewChainSelector].OffRamp, - e.Chains[cfg.NewChainSelector], + state.Chains[cfg.DONChainSelector].OffRamp, + e.Chains[cfg.DONChainSelector], + nodes.NonBootstraps(), + state.Chains[cfg.HomeChainSelector].RMNHome.Address(), + cfg.CCIPOCRParams.OCRParameters, + cfg.CCIPOCRParams.CommitOffChainConfig, + cfg.CCIPOCRParams.ExecuteOffChainConfig, + ) + if err != nil { + return deployment.ChangesetOutput{}, err + } + + latestDon, err := internal.LatestCCIPDON(state.Chains[cfg.HomeChainSelector].CapabilityRegistry) + if err != nil { + return deployment.ChangesetOutput{}, err + } + + pluginOCR3Config, ok := newDONArgs[cfg.PluginType] + if !ok { + return deployment.ChangesetOutput{}, fmt.Errorf("missing commit plugin in ocr3Configs") + } + + expectedDonID := latestDon.Id + 1 + addDonOp, err := newDonWithCandidateOp( + txOpts, + e.Chains[cfg.HomeChainSelector], + expectedDonID, + pluginOCR3Config, + state.Chains[cfg.HomeChainSelector].CapabilityRegistry, + nodes.NonBootstraps(), + cfg.MCMS != nil, + ) + if err != nil { + return deployment.ChangesetOutput{}, err + } + if cfg.MCMS == nil { + return deployment.ChangesetOutput{}, nil + } + + prop, err := proposalutils.BuildProposalFromBatches( + map[uint64]common.Address{ + cfg.HomeChainSelector: state.Chains[cfg.HomeChainSelector].Timelock.Address(), + }, + map[uint64]*gethwrappers.ManyChainMultiSig{ + cfg.HomeChainSelector: state.Chains[cfg.HomeChainSelector].ProposerMcm, + }, + []timelock.BatchChainOperation{{ + ChainIdentifier: mcms.ChainIdentifier(cfg.HomeChainSelector), + Batch: []mcms.Operation{addDonOp}, + }}, + fmt.Sprintf("addDON on new Chain && setCandidate for plugin %s", cfg.PluginType.String()), + cfg.MCMS.MinDelay, + ) + if err != nil { + return deployment.ChangesetOutput{}, fmt.Errorf("failed to build proposal from batch: %w", err) + } + + return deployment.ChangesetOutput{ + Proposals: []timelock.MCMSWithTimelockProposal{*prop}, + }, nil +} + +// newDonWithCandidateOp sets the candidate commit config by calling setCandidate on CCIPHome contract through the AddDON call on CapReg contract +// This should be done first before calling any other UpdateDON calls +// This proposes to set up OCR3 config for the commit plugin for the DON +func newDonWithCandidateOp( + txOpts *bind.TransactOpts, + homeChain deployment.Chain, + donID uint32, + pluginConfig ccip_home.CCIPHomeOCR3Config, + capReg *capabilities_registry.CapabilitiesRegistry, + nodes deployment.Nodes, + mcmsEnabled bool, +) (mcms.Operation, error) { + encodedSetCandidateCall, err := internal.CCIPHomeABI.Pack( + "setCandidate", + donID, + pluginConfig.PluginType, + pluginConfig, + [32]byte{}, + ) + if err != nil { + return mcms.Operation{}, fmt.Errorf("pack set candidate call: %w", err) + } + + addDonTx, err := capReg.AddDON( + txOpts, + nodes.PeerIDs(), + []capabilities_registry.CapabilitiesRegistryCapabilityConfiguration{ + { + CapabilityId: internal.CCIPCapabilityID, + Config: encodedSetCandidateCall, + }, + }, + false, // isPublic + false, // acceptsWorkflows + nodes.DefaultF(), + ) + if err != nil { + return mcms.Operation{}, fmt.Errorf("could not generate add don tx w/ commit config: %w", err) + } + if !mcmsEnabled { + _, err = deployment.ConfirmIfNoError(homeChain, addDonTx, err) + if err != nil { + return mcms.Operation{}, fmt.Errorf("error confirming addDon call: %w", err) + } + } + + return mcms.Operation{ + To: capReg.Address(), + Data: addDonTx.Data(), + Value: big.NewInt(0), + }, nil +} + +// SetCandidateChangeset generates a proposal to call setCandidate on the CCIPHome through the capability registry. +// A DON must exist in order to use this changeset effectively, i.e AddDonAndSetCandidateChangeset must be called first. +func SetCandidateChangeset( + e deployment.Environment, + cfg SetCandidateChangesetConfig, +) (deployment.ChangesetOutput, error) { + state, err := LoadOnchainState(e) + if err != nil { + return deployment.ChangesetOutput{}, err + } + + nodes, err := cfg.Validate(e, state) + if err != nil { + return deployment.ChangesetOutput{}, fmt.Errorf("%w: %w", deployment.ErrInvalidConfig, err) + } + + txOpts := e.Chains[cfg.HomeChainSelector].DeployerKey + if cfg.MCMS != nil { + txOpts = deployment.SimTransactOpts() + } + + newDONArgs, err := internal.BuildOCR3ConfigForCCIPHome( + e.OCRSecrets, + state.Chains[cfg.DONChainSelector].OffRamp, + e.Chains[cfg.DONChainSelector], nodes.NonBootstraps(), state.Chains[cfg.HomeChainSelector].RMNHome.Address(), cfg.CCIPOCRParams.OCRParameters, @@ -195,33 +466,37 @@ func SetCandidatePluginChangeset( } setCandidateMCMSOps, err := setCandidateOnExistingDon( + e.Logger, + txOpts, + e.Chains[cfg.HomeChainSelector], config, state.Chains[cfg.HomeChainSelector].CapabilityRegistry, state.Chains[cfg.HomeChainSelector].CCIPHome, - cfg.NewChainSelector, + cfg.DONChainSelector, nodes.NonBootstraps(), + cfg.MCMS != nil, ) if err != nil { return deployment.ChangesetOutput{}, err } - var ( - timelocksPerChain = map[uint64]common.Address{ + if cfg.MCMS == nil { + return deployment.ChangesetOutput{}, nil + } + + prop, err := proposalutils.BuildProposalFromBatches( + map[uint64]common.Address{ cfg.HomeChainSelector: state.Chains[cfg.HomeChainSelector].Timelock.Address(), - } - proposerMCMSes = map[uint64]*gethwrappers.ManyChainMultiSig{ + }, + map[uint64]*gethwrappers.ManyChainMultiSig{ cfg.HomeChainSelector: state.Chains[cfg.HomeChainSelector].ProposerMcm, - } - ) - prop, err := proposalutils.BuildProposalFromBatches( - timelocksPerChain, - proposerMCMSes, + }, []timelock.BatchChainOperation{{ ChainIdentifier: mcms.ChainIdentifier(cfg.HomeChainSelector), Batch: setCandidateMCMSOps, }}, fmt.Sprintf("SetCandidate for %s plugin", cfg.PluginType.String()), - 0, // minDelay + cfg.MCMS.MinDelay, ) if err != nil { return deployment.ChangesetOutput{}, err @@ -236,11 +511,15 @@ func SetCandidatePluginChangeset( // setCandidateOnExistingDon calls setCandidate on CCIPHome contract through the UpdateDON call on CapReg contract // This proposes to set up OCR3 config for the provided plugin for the DON func setCandidateOnExistingDon( + lggr logger.Logger, + txOpts *bind.TransactOpts, + homeChain deployment.Chain, pluginConfig ccip_home.CCIPHomeOCR3Config, capReg *capabilities_registry.CapabilitiesRegistry, ccipHome *ccip_home.CCIPHome, chainSelector uint64, nodes deployment.Nodes, + mcmsEnabled bool, ) ([]mcms.Operation, error) { // fetch DON ID for the chain donID, err := internal.DonIDForChain(capReg, ccipHome, chainSelector) @@ -251,7 +530,8 @@ func setCandidateOnExistingDon( return nil, fmt.Errorf("don doesn't exist in CR for chain %d", chainSelector) } - fmt.Printf("donID: %d", donID) + lggr.Infof("donID for chain %d: %d", chainSelector, donID) + encodedSetCandidateCall, err := internal.CCIPHomeABI.Pack( "setCandidate", donID, @@ -265,7 +545,7 @@ func setCandidateOnExistingDon( // set candidate call updateDonTx, err := capReg.UpdateDON( - deployment.SimTransactOpts(), + txOpts, donID, nodes.PeerIDs(), []capabilities_registry.CapabilitiesRegistryCapabilityConfiguration{ @@ -280,6 +560,12 @@ func setCandidateOnExistingDon( if err != nil { return nil, fmt.Errorf("update don w/ exec config: %w", err) } + if !mcmsEnabled { + _, err = deployment.ConfirmIfNoError(homeChain, updateDonTx, err) + if err != nil { + return nil, fmt.Errorf("error confirming updateDon call: %w", err) + } + } return []mcms.Operation{{ To: capReg.Address(), diff --git a/deployment/ccip/changeset/cs_ccip_home_test.go b/deployment/ccip/changeset/cs_ccip_home_test.go index 47f262d3f83..c4df4fe32d7 100644 --- a/deployment/ccip/changeset/cs_ccip_home_test.go +++ b/deployment/ccip/changeset/cs_ccip_home_test.go @@ -16,6 +16,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types" cctypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" + "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/deployment" @@ -164,11 +165,15 @@ func TestActiveCandidate(t *testing.T) { } ) setCommitCandidateOp, err := setCandidateOnExistingDon( + e.Logger, + deployment.SimTransactOpts(), + tenv.Env.Chains[tenv.HomeChainSel], ocr3ConfigMap[cctypes.PluginTypeCCIPCommit], state.Chains[tenv.HomeChainSel].CapabilityRegistry, state.Chains[tenv.HomeChainSel].CCIPHome, tenv.FeedChainSel, nodes.NonBootstraps(), + true, ) require.NoError(t, err) setCommitCandidateProposal, err := proposalutils.BuildProposalFromBatches(timelocksPerChain, proposerMCMSes, []timelock.BatchChainOperation{{ @@ -184,11 +189,15 @@ func TestActiveCandidate(t *testing.T) { // create the op for the commit plugin as well setExecCandidateOp, err := setCandidateOnExistingDon( + e.Logger, + deployment.SimTransactOpts(), + tenv.Env.Chains[tenv.HomeChainSel], ocr3ConfigMap[cctypes.PluginTypeCCIPExec], state.Chains[tenv.HomeChainSel].CapabilityRegistry, state.Chains[tenv.HomeChainSel].CCIPHome, tenv.FeedChainSel, nodes.NonBootstraps(), + true, ) require.NoError(t, err) @@ -288,37 +297,9 @@ func Test_PromoteCandidate(t *testing.T) { source := allChains[0] dest := allChains[1] - nodes, err := deployment.NodeInfo(tenv.Env.NodeIDs, tenv.Env.Offchain) - require.NoError(t, err) - - var nodeIDs []string - for _, node := range nodes { - nodeIDs = append(nodeIDs, node.NodeID) - } - if tc.mcmsEnabled { // Transfer ownership to timelock so that we can promote the zero digest later down the line. - _, err = commonchangeset.ApplyChangesets(t, tenv.Env, map[uint64]*proposalutils.TimelockExecutionContracts{ - source: { - Timelock: state.Chains[source].Timelock, - CallProxy: state.Chains[source].CallProxy, - }, - dest: { - Timelock: state.Chains[dest].Timelock, - CallProxy: state.Chains[dest].CallProxy, - }, - tenv.HomeChainSel: { - Timelock: state.Chains[tenv.HomeChainSel].Timelock, - CallProxy: state.Chains[tenv.HomeChainSel].CallProxy, - }, - }, []commonchangeset.ChangesetApplication{ - { - Changeset: commonchangeset.WrapChangeSet(commonchangeset.TransferToMCMSWithTimelock), - Config: genTestTransferOwnershipConfig(tenv, allChains, state), - }, - }) - require.NoError(t, err) - assertTimelockOwnership(t, tenv, allChains, state) + transferToTimelock(t, tenv, state, source, dest) } var ( @@ -356,7 +337,6 @@ func Test_PromoteCandidate(t *testing.T) { Config: PromoteAllCandidatesChangesetConfig{ HomeChainSelector: tenv.HomeChainSel, DONChainSelector: dest, - NodeIDs: nodeIDs, MCMS: mcmsConfig, }, }, @@ -378,3 +358,148 @@ func Test_PromoteCandidate(t *testing.T) { }) } } + +func Test_SetCandidate(t *testing.T) { + for _, tc := range []struct { + name string + mcmsEnabled bool + }{ + { + name: "MCMS enabled", + mcmsEnabled: true, + }, + { + name: "MCMS disabled", + mcmsEnabled: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := testcontext.Get(t) + tenv := NewMemoryEnvironment(t, + WithChains(2), + WithNodes(4)) + state, err := LoadOnchainState(tenv.Env) + require.NoError(t, err) + + // Deploy to all chains. + allChains := maps.Keys(tenv.Env.Chains) + source := allChains[0] + dest := allChains[1] + + if tc.mcmsEnabled { + // Transfer ownership to timelock so that we can promote the zero digest later down the line. + transferToTimelock(t, tenv, state, source, dest) + } + + var ( + capReg = state.Chains[tenv.HomeChainSel].CapabilityRegistry + ccipHome = state.Chains[tenv.HomeChainSel].CCIPHome + ) + donID, err := internal.DonIDForChain(capReg, ccipHome, dest) + require.NoError(t, err) + require.NotEqual(t, uint32(0), donID) + candidateDigestCommitBefore, err := ccipHome.GetCandidateDigest(&bind.CallOpts{ + Context: ctx, + }, donID, uint8(types.PluginTypeCCIPCommit)) + require.NoError(t, err) + require.Equal(t, [32]byte{}, candidateDigestCommitBefore) + candidateDigestExecBefore, err := ccipHome.GetCandidateDigest(&bind.CallOpts{ + Context: ctx, + }, donID, uint8(types.PluginTypeCCIPExec)) + require.NoError(t, err) + require.Equal(t, [32]byte{}, candidateDigestExecBefore) + + var mcmsConfig *MCMSConfig + if tc.mcmsEnabled { + mcmsConfig = &MCMSConfig{ + MinDelay: 0, + } + } + tokenConfig := NewTestTokenConfig(state.Chains[tenv.FeedChainSel].USDFeeds) + _, err = commonchangeset.ApplyChangesets(t, tenv.Env, map[uint64]*proposalutils.TimelockExecutionContracts{ + tenv.HomeChainSel: { + Timelock: state.Chains[tenv.HomeChainSel].Timelock, + CallProxy: state.Chains[tenv.HomeChainSel].CallProxy, + }, + }, []commonchangeset.ChangesetApplication{ + { + Changeset: commonchangeset.WrapChangeSet(SetCandidateChangeset), + Config: SetCandidateChangesetConfig{ + HomeChainSelector: tenv.HomeChainSel, + FeedChainSelector: tenv.FeedChainSel, + DONChainSelector: dest, + PluginType: types.PluginTypeCCIPCommit, + CCIPOCRParams: DefaultOCRParams( + tenv.FeedChainSel, + tokenConfig.GetTokenInfo(logger.TestLogger(t), state.Chains[dest].LinkToken, state.Chains[dest].Weth9), + nil, + ), + MCMS: mcmsConfig, + }, + }, + { + Changeset: commonchangeset.WrapChangeSet(SetCandidateChangeset), + Config: SetCandidateChangesetConfig{ + HomeChainSelector: tenv.HomeChainSel, + FeedChainSelector: tenv.FeedChainSel, + DONChainSelector: dest, + PluginType: types.PluginTypeCCIPExec, + CCIPOCRParams: DefaultOCRParams( + tenv.FeedChainSel, + tokenConfig.GetTokenInfo(logger.TestLogger(t), state.Chains[dest].LinkToken, state.Chains[dest].Weth9), + nil, + ), + MCMS: mcmsConfig, + }, + }, + }) + require.NoError(t, err) + + // after setting a new candidate on both plugins, the candidate config digest + // should be nonzero. + candidateDigestCommitAfter, err := ccipHome.GetCandidateDigest(&bind.CallOpts{ + Context: ctx, + }, donID, uint8(types.PluginTypeCCIPCommit)) + require.NoError(t, err) + require.NotEqual(t, [32]byte{}, candidateDigestCommitAfter) + require.NotEqual(t, candidateDigestCommitBefore, candidateDigestCommitAfter) + + candidateDigestExecAfter, err := ccipHome.GetCandidateDigest(&bind.CallOpts{ + Context: ctx, + }, donID, uint8(types.PluginTypeCCIPExec)) + require.NoError(t, err) + require.NotEqual(t, [32]byte{}, candidateDigestExecAfter) + require.NotEqual(t, candidateDigestExecBefore, candidateDigestExecAfter) + }) + } +} + +func transferToTimelock( + t *testing.T, + tenv DeployedEnv, + state CCIPOnChainState, + source, + dest uint64) { + // Transfer ownership to timelock so that we can promote the zero digest later down the line. + _, err := commonchangeset.ApplyChangesets(t, tenv.Env, map[uint64]*proposalutils.TimelockExecutionContracts{ + source: { + Timelock: state.Chains[source].Timelock, + CallProxy: state.Chains[source].CallProxy, + }, + dest: { + Timelock: state.Chains[dest].Timelock, + CallProxy: state.Chains[dest].CallProxy, + }, + tenv.HomeChainSel: { + Timelock: state.Chains[tenv.HomeChainSel].Timelock, + CallProxy: state.Chains[tenv.HomeChainSel].CallProxy, + }, + }, []commonchangeset.ChangesetApplication{ + { + Changeset: commonchangeset.WrapChangeSet(commonchangeset.TransferToMCMSWithTimelock), + Config: genTestTransferOwnershipConfig(tenv, []uint64{source, dest}, state), + }, + }) + require.NoError(t, err) + assertTimelockOwnership(t, tenv, []uint64{source, dest}, state) +} diff --git a/deployment/ccip/changeset/cs_initial_add_chain.go b/deployment/ccip/changeset/cs_initial_add_chain.go index 5ba648d74b5..4f8b2ac2722 100644 --- a/deployment/ccip/changeset/cs_initial_add_chain.go +++ b/deployment/ccip/changeset/cs_initial_add_chain.go @@ -483,7 +483,7 @@ func ValidateCCIPHomeConfigSetUp( return fmt.Errorf("fetch don id for chain: %w", err) } if donID == 0 { - return fmt.Errorf("don id for chain(%d) does not exist", chainSel) + return fmt.Errorf("don id for chain (%d) does not exist", chainSel) } // final sanity checks on configs. From 7ca4930c1cf931b4b6d4d45c223a62e00c5edb7e Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 12 Dec 2024 19:08:46 -0500 Subject: [PATCH 08/11] Attempt fix for duplicate metrics collector error (#15663) * Attempt fix for duplicate metrics collector error * Fix thread busy metrics in llo mercury transmitter --- .../services/llo/mercurytransmitter/server.go | 59 +++++++++++++++---- .../llo/mercurytransmitter/transmitter.go | 34 +---------- 2 files changed, 49 insertions(+), 44 deletions(-) diff --git a/core/services/llo/mercurytransmitter/server.go b/core/services/llo/mercurytransmitter/server.go index 4e97c0483b3..3ce2b0a4b4a 100644 --- a/core/services/llo/mercurytransmitter/server.go +++ b/core/services/llo/mercurytransmitter/server.go @@ -62,6 +62,22 @@ var ( }, []string{"donID", "serverURL", "code"}, ) + promTransmitConcurrentTransmitGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "llo", + Subsystem: "mercurytransmitter", + Name: "concurrent_transmit_gauge", + Help: "Gauge that measures the number of transmit threads currently waiting on a remote transmit call. You may wish to alert if this exceeds some number for a given period of time, or if it ever reaches its max.", + }, + []string{"donID", "serverURL"}, + ) + promTransmitConcurrentDeleteGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "llo", + Subsystem: "mercurytransmitter", + Name: "concurrent_delete_gauge", + Help: "Gauge that measures the number of delete threads currently waiting on a delete call to the DB. You may wish to alert if this exceeds some number for a given period of time, or if it ever reaches its max.", + }, + []string{"donID", "serverURL"}, + ) ) type ReportPacker interface { @@ -87,12 +103,14 @@ type server struct { evmPremiumLegacyPacker ReportPacker jsonPacker ReportPacker - transmitSuccessCount prometheus.Counter - transmitDuplicateCount prometheus.Counter - transmitConnectionErrorCount prometheus.Counter - transmitQueueDeleteErrorCount prometheus.Counter - transmitQueueInsertErrorCount prometheus.Counter - transmitQueuePushErrorCount prometheus.Counter + transmitSuccessCount prometheus.Counter + transmitDuplicateCount prometheus.Counter + transmitConnectionErrorCount prometheus.Counter + transmitQueueDeleteErrorCount prometheus.Counter + transmitQueueInsertErrorCount prometheus.Counter + transmitQueuePushErrorCount prometheus.Counter + transmitConcurrentTransmitGauge prometheus.Gauge + transmitConcurrentDeleteGauge prometheus.Gauge transmitThreadBusyCount atomic.Int32 deleteThreadBusyCount atomic.Int32 @@ -130,6 +148,8 @@ func newServer(lggr logger.Logger, verboseLogging bool, cfg QueueConfig, client promTransmitQueueDeleteErrorCount.WithLabelValues(donIDStr, serverURL), promTransmitQueueInsertErrorCount.WithLabelValues(donIDStr, serverURL), promTransmitQueuePushErrorCount.WithLabelValues(donIDStr, serverURL), + promTransmitConcurrentTransmitGauge.WithLabelValues(donIDStr, serverURL), + promTransmitConcurrentDeleteGauge.WithLabelValues(donIDStr, serverURL), atomic.Int32{}, atomic.Int32{}, } @@ -161,7 +181,7 @@ func (s *server) runDeleteQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup select { case hash := <-s.deleteQueue: for { - s.deleteThreadBusyCount.Add(1) + s.deleteThreadBusyCountInc() if err := s.pm.orm.Delete(ctx, [][32]byte{hash}); err != nil { s.lggr.Errorw("Failed to delete transmission record", "err", err, "transmissionHash", hash) s.transmitQueueDeleteErrorCount.Inc() @@ -170,7 +190,7 @@ func (s *server) runDeleteQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup // Wait a backoff duration before trying to delete again continue case <-stopCh: - s.deleteThreadBusyCount.Add(-1) + s.deleteThreadBusyCountDec() // abort and return immediately on stop even if items remain in queue return } @@ -179,7 +199,7 @@ func (s *server) runDeleteQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup } // success b.Reset() - s.deleteThreadBusyCount.Add(-1) + s.deleteThreadBusyCountDec() case <-stopCh: // abort and return immediately on stop even if items remain in queue return @@ -187,6 +207,23 @@ func (s *server) runDeleteQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup } } +func (s *server) transmitThreadBusyCountInc() { + val := s.transmitThreadBusyCount.Add(1) + s.transmitConcurrentTransmitGauge.Set(float64(val)) +} +func (s *server) transmitThreadBusyCountDec() { + val := s.transmitThreadBusyCount.Add(-1) + s.transmitConcurrentTransmitGauge.Set(float64(val)) +} +func (s *server) deleteThreadBusyCountInc() { + val := s.deleteThreadBusyCount.Add(1) + s.transmitConcurrentDeleteGauge.Set(float64(val)) +} +func (s *server) deleteThreadBusyCountDec() { + val := s.deleteThreadBusyCount.Add(-1) + s.transmitConcurrentDeleteGauge.Set(float64(val)) +} + func (s *server) runQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup, donIDStr string) { defer wg.Done() // Exponential backoff with very short retry interval (since latency is a priority) @@ -208,8 +245,8 @@ func (s *server) runQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup, donI return false } - s.transmitThreadBusyCount.Add(1) - defer s.transmitThreadBusyCount.Add(-1) + s.transmitThreadBusyCountInc() + defer s.transmitThreadBusyCountDec() req, res, err := func(ctx context.Context) (*pb.TransmitRequest, *pb.TransmitResponse, error) { ctx, cancelFn := context.WithTimeout(ctx, utils.WithJitter(s.transmitTimeout)) diff --git a/core/services/llo/mercurytransmitter/transmitter.go b/core/services/llo/mercurytransmitter/transmitter.go index 8e60bf938a5..23aa4b79e58 100644 --- a/core/services/llo/mercurytransmitter/transmitter.go +++ b/core/services/llo/mercurytransmitter/transmitter.go @@ -116,7 +116,6 @@ type transmitter struct { orm ORM servers map[string]*server registerer prometheus.Registerer - collectors []prometheus.Collector donID uint32 fromAccount string @@ -155,7 +154,6 @@ func newTransmitter(opts Opts) *transmitter { opts.ORM, servers, opts.Registerer, - nil, opts.DonID, fmt.Sprintf("%x", opts.FromAccount), make(services.StopChan), @@ -194,31 +192,6 @@ func (mt *transmitter) Start(ctx context.Context) (err error) { go s.runDeleteQueueLoop(mt.stopCh, mt.wg) go s.runQueueLoop(mt.stopCh, mt.wg, donIDStr) } - mt.collectors = append(mt.collectors, prometheus.NewGaugeFunc( - prometheus.GaugeOpts{ - Namespace: "llo", - Subsystem: "mercurytransmitter", - Name: "concurrent_transmit_gauge", - Help: "Gauge that measures the number of transmit threads currently waiting on a remote transmit call. You may wish to alert if this exceeds some number for a given period of time, or if it ever reaches its max.", - ConstLabels: prometheus.Labels{"donID": donIDStr, "serverURL": s.url, "maxConcurrentTransmits": strconv.FormatInt(int64(nThreads), 10)}, - }, func() float64 { - return float64(s.transmitThreadBusyCount.Load()) - })) - mt.collectors = append(mt.collectors, prometheus.NewGaugeFunc( - prometheus.GaugeOpts{ - Namespace: "llo", - Subsystem: "mercurytransmitter", - Name: "concurrent_delete_gauge", - Help: "Gauge that measures the number of delete threads currently waiting on a delete call to the DB. You may wish to alert if this exceeds some number for a given period of time, or if it ever reaches its max.", - ConstLabels: prometheus.Labels{"donID": donIDStr, "serverURL": s.url, "maxConcurrentDeletes": strconv.FormatInt(int64(nThreads), 10)}, - }, func() float64 { - return float64(s.deleteThreadBusyCount.Load()) - })) - for _, c := range mt.collectors { - if err := mt.registerer.Register(c); err != nil { - return err - } - } } if err := (&services.MultiStart{}).Start(ctx, startClosers...); err != nil { return err @@ -250,12 +223,7 @@ func (mt *transmitter) Close() error { closers = append(closers, s.pm) closers = append(closers, s.c) } - err := services.CloseAll(closers...) - // Unregister all the gauge funcs - for _, c := range mt.collectors { - mt.registerer.Unregister(c) - } - return err + return services.CloseAll(closers...) }) } From 97b05638bd9c69a7a1cf90bb929811a260247cb1 Mon Sep 17 00:00:00 2001 From: Pablo Estrada <139084212+ecPablo@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:28:08 -0600 Subject: [PATCH 09/11] feat: link transfer mcms changesets (#15512) * feat: link transfer with timelock changeset * feat: link transfer and approval integration tests and changesets. * feat: rename files * fix: use deployment.SimTransactOpts() to get tx data * fix: link contract creation * fix: remove approval changeset, not necessary for sending directly from the owner * feat: make config accept a map of chain selectors for the proposal generation * fix: params on deploy link * fix: simplify config args by using state helper functions. * fix: use pointer for value * feat: add mint permissions and minting link changeset * Deploy call proxy instead of using deployer executor keys * inject call proxies in execution methods * skip call proxy when loading chain state * revert all changes * Revert "revert all changes" This reverts commit c17911eb1ff4382b3f80d0edb055d748096dc59f. * chore: rename load state funcs * feat: add mcms config flag * fix: integration tests after merging develop * fix: use contracts from states and code improvements * fix: cs deploy chain args * fix: params ccip boosting * fix: bundle mcms config into single struct * fix: add more validations for config * fix: remove startingOpCount and use proposal utils to derive it * fix: adjust variable names, remove boolean for mcms config, add constants move balance validation to Validate() function * feat: add tests for non mcms case, improve validations, and wait for tx confirmation. * feat: check valid until is in future * feat: add tests for Validate() and small validation fixes * fix: rename MaybeLoadLinkTokenState to MaybeLoadLinkTokenChainState to abstract loading state per chain * Update deployment/common/changeset/example/link_transfer.go Co-authored-by: Graham Goh * fix: error handling and validations * fix: use getDeployer helper * feat: split mint burners into a separate changeset * fix: name TestMintLink on unit test * Update deployment/common/changeset/example/add_mint_burners_link.go Co-authored-by: Graham Goh * Update deployment/common/changeset/example/add_mint_burners_link.go Co-authored-by: Graham Goh * fix: use changeset apply for unit tests environment setup * fix: linting errors * fix: merge conflicts * fix: remove valid unit to reuse util for proposal creation --------- Co-authored-by: Akhil Chainani Co-authored-by: Graham Goh --- deployment/ccip/changeset/state.go | 4 +- .../common/changeset/deploy_link_token.go | 2 +- .../changeset/deploy_link_token_test.go | 2 +- .../example/add_mint_burners_link.go | 70 ++++ .../example/add_mint_burners_link_test.go | 50 +++ .../common/changeset/example/link_transfer.go | 239 +++++++++++ .../changeset/example/link_transfer_test.go | 373 ++++++++++++++++++ .../common/changeset/example/mint_link.go | 43 ++ .../changeset/example/mint_link_test.go | 58 +++ .../common/changeset/internal/mcms_test.go | 2 +- deployment/common/changeset/state.go | 120 +++++- .../transfer_to_mcms_with_timelock_test.go | 6 +- deployment/common/proposalutils/propose.go | 5 +- deployment/environment.go | 2 +- .../changeset/accept_ownership_test.go | 2 +- deployment/keystone/state.go | 2 +- 16 files changed, 955 insertions(+), 25 deletions(-) create mode 100644 deployment/common/changeset/example/add_mint_burners_link.go create mode 100644 deployment/common/changeset/example/add_mint_burners_link_test.go create mode 100644 deployment/common/changeset/example/link_transfer.go create mode 100644 deployment/common/changeset/example/link_transfer_test.go create mode 100644 deployment/common/changeset/example/mint_link.go create mode 100644 deployment/common/changeset/example/mint_link_test.go diff --git a/deployment/ccip/changeset/state.go b/deployment/ccip/changeset/state.go index 45ea9e8f5b8..cd88db1b9ee 100644 --- a/deployment/ccip/changeset/state.go +++ b/deployment/ccip/changeset/state.go @@ -311,13 +311,13 @@ func LoadOnchainState(e deployment.Environment) (CCIPOnChainState, error) { // LoadChainState Loads all state for a chain into state func LoadChainState(chain deployment.Chain, addresses map[string]deployment.TypeAndVersion) (CCIPChainState, error) { var state CCIPChainState - mcmsWithTimelock, err := commoncs.MaybeLoadMCMSWithTimelockState(chain, addresses) + mcmsWithTimelock, err := commoncs.MaybeLoadMCMSWithTimelockChainState(chain, addresses) if err != nil { return state, err } state.MCMSWithTimelockState = *mcmsWithTimelock - linkState, err := commoncs.MaybeLoadLinkTokenState(chain, addresses) + linkState, err := commoncs.MaybeLoadLinkTokenChainState(chain, addresses) if err != nil { return state, err } diff --git a/deployment/common/changeset/deploy_link_token.go b/deployment/common/changeset/deploy_link_token.go index 292c07c93df..c115a7ee083 100644 --- a/deployment/common/changeset/deploy_link_token.go +++ b/deployment/common/changeset/deploy_link_token.go @@ -12,7 +12,7 @@ import ( var _ deployment.ChangeSet[[]uint64] = DeployLinkToken -// DeployLinkToken deploys a link token contract to the chain identified by the chainSelector. +// DeployLinkToken deploys a link token contract to the chain identified by the ChainSelector. func DeployLinkToken(e deployment.Environment, chains []uint64) (deployment.ChangesetOutput, error) { for _, chain := range chains { _, ok := e.Chains[chain] diff --git a/deployment/common/changeset/deploy_link_token_test.go b/deployment/common/changeset/deploy_link_token_test.go index a61743e9bf4..bc472d2a247 100644 --- a/deployment/common/changeset/deploy_link_token_test.go +++ b/deployment/common/changeset/deploy_link_token_test.go @@ -27,7 +27,7 @@ func TestDeployLinkToken(t *testing.T) { require.NoError(t, err) addrs, err := e.ExistingAddresses.AddressesForChain(chain1) require.NoError(t, err) - state, err := changeset.MaybeLoadLinkTokenState(e.Chains[chain1], addrs) + state, err := changeset.MaybeLoadLinkTokenChainState(e.Chains[chain1], addrs) require.NoError(t, err) // View itself already unit tested _, err = state.GenerateLinkView() diff --git a/deployment/common/changeset/example/add_mint_burners_link.go b/deployment/common/changeset/example/add_mint_burners_link.go new file mode 100644 index 00000000000..7322f99dd60 --- /dev/null +++ b/deployment/common/changeset/example/add_mint_burners_link.go @@ -0,0 +1,70 @@ +package example + +import ( + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/common/changeset" +) + +type AddMintersBurnersLinkConfig struct { + ChainSelector uint64 + Minters []common.Address + Burners []common.Address +} + +var _ deployment.ChangeSet[*AddMintersBurnersLinkConfig] = AddMintersBurnersLink + +// AddMintersBurnersLink grants the minter / burner role to the provided addresses. +func AddMintersBurnersLink(e deployment.Environment, cfg *AddMintersBurnersLinkConfig) (deployment.ChangesetOutput, error) { + + chain := e.Chains[cfg.ChainSelector] + addresses, err := e.ExistingAddresses.AddressesForChain(cfg.ChainSelector) + if err != nil { + return deployment.ChangesetOutput{}, err + } + linkState, err := changeset.MaybeLoadLinkTokenChainState(chain, addresses) + if err != nil { + return deployment.ChangesetOutput{}, err + } + + for _, minter := range cfg.Minters { + // check if minter is already a minter + isMinter, err := linkState.LinkToken.IsMinter(&bind.CallOpts{Context: e.GetContext()}, minter) + if err != nil { + return deployment.ChangesetOutput{}, err + } + if isMinter { + continue + } + tx, err := linkState.LinkToken.GrantMintRole(chain.DeployerKey, minter) + if err != nil { + return deployment.ChangesetOutput{}, err + } + _, err = deployment.ConfirmIfNoError(chain, tx, err) + if err != nil { + return deployment.ChangesetOutput{}, err + } + } + for _, burner := range cfg.Burners { + // check if burner is already a burner + isBurner, err := linkState.LinkToken.IsBurner(&bind.CallOpts{Context: e.GetContext()}, burner) + if err != nil { + return deployment.ChangesetOutput{}, err + } + if isBurner { + continue + } + tx, err := linkState.LinkToken.GrantBurnRole(chain.DeployerKey, burner) + if err != nil { + return deployment.ChangesetOutput{}, err + } + _, err = deployment.ConfirmIfNoError(chain, tx, err) + if err != nil { + return deployment.ChangesetOutput{}, err + } + } + return deployment.ChangesetOutput{}, nil + +} diff --git a/deployment/common/changeset/example/add_mint_burners_link_test.go b/deployment/common/changeset/example/add_mint_burners_link_test.go new file mode 100644 index 00000000000..4dbfddc0b30 --- /dev/null +++ b/deployment/common/changeset/example/add_mint_burners_link_test.go @@ -0,0 +1,50 @@ +package example_test + +import ( + "context" + "testing" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/deployment/common/changeset" + "github.com/smartcontractkit/chainlink/deployment/common/changeset/example" +) + +// TestAddMintersBurnersLink tests the AddMintersBurnersLink changeset +func TestAddMintersBurnersLink(t *testing.T) { + t.Parallel() + ctx := context.Background() + // Deploy Link Token and Timelock contracts and add addresses to environment + env := setupLinkTransferTestEnv(t) + + chainSelector := env.AllChainSelectors()[0] + chain := env.Chains[chainSelector] + addrs, err := env.ExistingAddresses.AddressesForChain(chainSelector) + require.NoError(t, err) + require.Len(t, addrs, 6) + + mcmsState, err := changeset.MaybeLoadMCMSWithTimelockChainState(chain, addrs) + require.NoError(t, err) + linkState, err := changeset.MaybeLoadLinkTokenChainState(chain, addrs) + require.NoError(t, err) + + timelockAddress := mcmsState.Timelock.Address() + + // Mint some funds + _, err = example.AddMintersBurnersLink(env, &example.AddMintersBurnersLinkConfig{ + ChainSelector: chainSelector, + Minters: []common.Address{timelockAddress}, + Burners: []common.Address{timelockAddress}, + }) + require.NoError(t, err) + + // check timelock balance + isMinter, err := linkState.LinkToken.IsMinter(&bind.CallOpts{Context: ctx}, timelockAddress) + require.NoError(t, err) + require.True(t, isMinter) + isBurner, err := linkState.LinkToken.IsBurner(&bind.CallOpts{Context: ctx}, timelockAddress) + require.NoError(t, err) + require.True(t, isBurner) +} diff --git a/deployment/common/changeset/example/link_transfer.go b/deployment/common/changeset/example/link_transfer.go new file mode 100644 index 00000000000..2e3be48a4d1 --- /dev/null +++ b/deployment/common/changeset/example/link_transfer.go @@ -0,0 +1,239 @@ +package example + +import ( + "errors" + "fmt" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + ethTypes "github.com/ethereum/go-ethereum/core/types" + owner_helpers "github.com/smartcontractkit/ccip-owner-contracts/pkg/gethwrappers" + chain_selectors "github.com/smartcontractkit/chain-selectors" + + "github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/mcms" + "github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock" + + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/common/changeset" + "github.com/smartcontractkit/chainlink/deployment/common/proposalutils" + "github.com/smartcontractkit/chainlink/deployment/common/types" +) + +const MaxTimelockDelay = 24 * 7 * time.Hour + +type TransferConfig struct { + To common.Address + Value *big.Int +} + +type MCMSConfig struct { + MinDelay time.Duration // delay for timelock worker to execute the transfers. + OverrideRoot bool +} + +type LinkTransferConfig struct { + Transfers map[uint64][]TransferConfig + From common.Address + McmsConfig *MCMSConfig +} + +var _ deployment.ChangeSet[*LinkTransferConfig] = LinkTransfer + +func getDeployer(e deployment.Environment, chain uint64, mcmConfig *MCMSConfig) *bind.TransactOpts { + if mcmConfig == nil { + return e.Chains[chain].DeployerKey + } + + return deployment.SimTransactOpts() +} + +// Validate checks that the LinkTransferConfig is valid. +func (cfg LinkTransferConfig) Validate(e deployment.Environment) error { + ctx := e.GetContext() + // Check that Transfers map has at least one chainSel + if len(cfg.Transfers) == 0 { + return errors.New("transfers map must have at least one chainSel") + } + + // Check transfers config values. + for chainSel, transfers := range cfg.Transfers { + selector, err := chain_selectors.GetSelectorFamily(chainSel) + if err != nil { + return fmt.Errorf("invalid chain selector: %w", err) + } + if selector != chain_selectors.FamilyEVM { + return fmt.Errorf("chain selector %d is not an EVM chain", chainSel) + } + chain, ok := e.Chains[chainSel] + if !ok { + return fmt.Errorf("chain with selector %d not found", chainSel) + } + addrs, err := e.ExistingAddresses.AddressesForChain(chainSel) + if err != nil { + return fmt.Errorf("error getting addresses for chain %d: %w", chainSel, err) + } + if len(transfers) == 0 { + return fmt.Errorf("transfers for chainSel %d must have at least one LinkTransfer", chainSel) + } + totalAmount := big.NewInt(0) + linkState, err := changeset.MaybeLoadLinkTokenChainState(chain, addrs) + if err != nil { + return fmt.Errorf("error loading link token state during validation: %w", err) + } + for _, transfer := range transfers { + if transfer.To == (common.Address{}) { + return errors.New("'to' address for transfers must be set") + } + if transfer.Value == nil { + return errors.New("value for transfers must be set") + } + if transfer.Value.Cmp(big.NewInt(0)) == 0 { + return errors.New("value for transfers must be non-zero") + } + if transfer.Value.Cmp(big.NewInt(0)) == -1 { + return errors.New("value for transfers must be positive") + } + totalAmount.Add(totalAmount, transfer.Value) + } + // check that from address has enough funds for the transfers + balance, err := linkState.LinkToken.BalanceOf(&bind.CallOpts{Context: ctx}, cfg.From) + if balance.Cmp(totalAmount) < 0 { + return fmt.Errorf("sender does not have enough funds for transfers for chain selector %d, required: %s, available: %s", chainSel, totalAmount.String(), balance.String()) + } + } + + if cfg.McmsConfig == nil { + return nil + } + + // Upper bound for min delay (7 days) + if cfg.McmsConfig.MinDelay > MaxTimelockDelay { + return errors.New("minDelay must be less than 7 days") + } + + return nil +} + +// initStatePerChain initializes the state for each chain selector on the provided config +func initStatePerChain(cfg *LinkTransferConfig, e deployment.Environment) ( + linkStatePerChain map[uint64]*changeset.LinkTokenState, + mcmsStatePerChain map[uint64]*changeset.MCMSWithTimelockState, + err error) { + linkStatePerChain = map[uint64]*changeset.LinkTokenState{} + mcmsStatePerChain = map[uint64]*changeset.MCMSWithTimelockState{} + // Load state for each chain + chainSelectors := []uint64{} + for chainSelector := range cfg.Transfers { + chainSelectors = append(chainSelectors, chainSelector) + } + linkStatePerChain, err = changeset.MaybeLoadLinkTokenState(e, chainSelectors) + if err != nil { + return nil, nil, err + } + mcmsStatePerChain, err = changeset.MaybeLoadMCMSWithTimelockState(e, chainSelectors) + if err != nil { + return nil, nil, err + + } + return linkStatePerChain, mcmsStatePerChain, nil +} + +// transferOrBuildTx transfers the LINK tokens or builds the tx for the MCMS proposal +func transferOrBuildTx( + e deployment.Environment, + linkState *changeset.LinkTokenState, + transfer TransferConfig, + opts *bind.TransactOpts, + chain deployment.Chain, + mcmsConfig *MCMSConfig) (*ethTypes.Transaction, error) { + tx, err := linkState.LinkToken.Transfer(opts, transfer.To, transfer.Value) + if err != nil { + return nil, fmt.Errorf("error packing transfer tx data: %w", err) + } + // only wait for tx if we are not using MCMS + if mcmsConfig == nil { + if _, err := deployment.ConfirmIfNoError(chain, tx, err); err != nil { + e.Logger.Errorw("Failed to confirm transfer tx", "chain", chain.String(), "err", err) + return nil, err + } + } + return tx, nil + +} + +// LinkTransfer takes the given link transfers and executes them or creates an MCMS proposal for them. +func LinkTransfer(e deployment.Environment, cfg *LinkTransferConfig) (deployment.ChangesetOutput, error) { + + err := cfg.Validate(e) + if err != nil { + return deployment.ChangesetOutput{}, fmt.Errorf("invalid LinkTransferConfig: %w", err) + } + chainSelectors := []uint64{} + for chainSelector := range cfg.Transfers { + chainSelectors = append(chainSelectors, chainSelector) + } + mcmsPerChain := map[uint64]*owner_helpers.ManyChainMultiSig{} + + timelockAddresses := map[uint64]common.Address{} + // Initialize state for each chain + linkStatePerChain, mcmsStatePerChain, err := initStatePerChain(cfg, e) + + allBatches := []timelock.BatchChainOperation{} + for chainSelector := range cfg.Transfers { + chainID := mcms.ChainIdentifier(chainSelector) + chain := e.Chains[chainSelector] + linkAddress := linkStatePerChain[chainSelector].LinkToken.Address() + mcmsState := mcmsStatePerChain[chainSelector] + linkState := linkStatePerChain[chainSelector] + + timelockAddress := mcmsState.Timelock.Address() + + mcmsPerChain[uint64(chainID)] = mcmsState.ProposerMcm + + timelockAddresses[chainSelector] = timelockAddress + batch := timelock.BatchChainOperation{ + ChainIdentifier: chainID, + Batch: []mcms.Operation{}, + } + + opts := getDeployer(e, chainSelector, cfg.McmsConfig) + totalAmount := big.NewInt(0) + for _, transfer := range cfg.Transfers[chainSelector] { + tx, err := transferOrBuildTx(e, linkState, transfer, opts, chain, cfg.McmsConfig) + if err != nil { + return deployment.ChangesetOutput{}, err + } + op := mcms.Operation{ + To: linkAddress, + Data: tx.Data(), + Value: big.NewInt(0), + ContractType: string(types.LinkToken), + } + batch.Batch = append(batch.Batch, op) + totalAmount.Add(totalAmount, transfer.Value) + } + + allBatches = append(allBatches, batch) + } + + if cfg.McmsConfig != nil { + proposal, err := proposalutils.BuildProposalFromBatches( + timelockAddresses, + mcmsPerChain, + allBatches, + "LINK Value transfer proposal", + cfg.McmsConfig.MinDelay, + ) + if err != nil { + return deployment.ChangesetOutput{}, err + } + + return deployment.ChangesetOutput{ + Proposals: []timelock.MCMSWithTimelockProposal{*proposal}, + }, nil + } + + return deployment.ChangesetOutput{}, nil +} diff --git a/deployment/common/changeset/example/link_transfer_test.go b/deployment/common/changeset/example/link_transfer_test.go new file mode 100644 index 00000000000..eecfbd37c95 --- /dev/null +++ b/deployment/common/changeset/example/link_transfer_test.go @@ -0,0 +1,373 @@ +package example_test + +import ( + "context" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + chain_selectors "github.com/smartcontractkit/chain-selectors" + + "github.com/smartcontractkit/chainlink/deployment/common/changeset/example" + "github.com/smartcontractkit/chainlink/deployment/common/proposalutils" + "github.com/smartcontractkit/chainlink/v2/core/logger" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/common/changeset" + "github.com/smartcontractkit/chainlink/deployment/common/types" + "github.com/smartcontractkit/chainlink/deployment/environment/memory" +) + +// setupLinkTransferContracts deploys all required contracts for the link transfer tests and returns the updated env. +func setupLinkTransferTestEnv(t *testing.T) deployment.Environment { + + lggr := logger.TestLogger(t) + cfg := memory.MemoryEnvironmentConfig{ + Nodes: 1, + Chains: 2, + } + env := memory.NewMemoryEnvironment(t, lggr, zapcore.DebugLevel, cfg) + chainSelector := env.AllChainSelectors()[0] + config := proposalutils.SingleGroupMCMS(t) + + // Deploy MCMS and Timelock + env, err := changeset.ApplyChangesets(t, env, nil, []changeset.ChangesetApplication{ + { + Changeset: changeset.WrapChangeSet(changeset.DeployLinkToken), + Config: []uint64{chainSelector}, + }, + { + Changeset: changeset.WrapChangeSet(changeset.DeployMCMSWithTimelock), + Config: map[uint64]types.MCMSWithTimelockConfig{ + chainSelector: { + Canceller: config, + Bypasser: config, + Proposer: config, + TimelockMinDelay: big.NewInt(0), + }, + }, + }, + }) + require.NoError(t, err) + return env +} + +// TestLinkTransferMCMS tests the LinkTransfer changeset by sending LINK from a timelock contract +// to the deployer key via mcms proposal. +func TestLinkTransferMCMS(t *testing.T) { + t.Parallel() + ctx := context.Background() + + env := setupLinkTransferTestEnv(t) + chainSelector := env.AllChainSelectors()[0] + chain := env.Chains[chainSelector] + addrs, err := env.ExistingAddresses.AddressesForChain(chainSelector) + require.NoError(t, err) + require.Len(t, addrs, 6) + + mcmsState, err := changeset.MaybeLoadMCMSWithTimelockChainState(chain, addrs) + require.NoError(t, err) + linkState, err := changeset.MaybeLoadLinkTokenChainState(chain, addrs) + require.NoError(t, err) + timelockAddress := mcmsState.Timelock.Address() + + // Mint some funds + // grant minter permissions + tx, err := linkState.LinkToken.GrantMintRole(chain.DeployerKey, chain.DeployerKey.From) + require.NoError(t, err) + _, err = deployment.ConfirmIfNoError(chain, tx, err) + require.NoError(t, err) + + tx, err = linkState.LinkToken.Mint(chain.DeployerKey, timelockAddress, big.NewInt(750)) + require.NoError(t, err) + _, err = deployment.ConfirmIfNoError(chain, tx, err) + require.NoError(t, err) + + timelocks := map[uint64]*proposalutils.TimelockExecutionContracts{ + chainSelector: { + Timelock: mcmsState.Timelock, + CallProxy: mcmsState.CallProxy, + }, + } + // Apply the changeset + _, err = changeset.ApplyChangesets(t, env, timelocks, []changeset.ChangesetApplication{ + // the changeset produces proposals, ApplyChangesets will sign & execute them. + // in practice, signing and executing are separated processes. + { + Changeset: changeset.WrapChangeSet(example.LinkTransfer), + Config: &example.LinkTransferConfig{ + From: timelockAddress, + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: { + { + To: chain.DeployerKey.From, + Value: big.NewInt(500), + }, + }, + }, + McmsConfig: &example.MCMSConfig{ + MinDelay: 0, + OverrideRoot: true, + }, + }, + }, + }) + require.NoError(t, err) + + // Check new balances + endBalance, err := linkState.LinkToken.BalanceOf(&bind.CallOpts{Context: ctx}, chain.DeployerKey.From) + require.NoError(t, err) + expectedBalance := big.NewInt(500) + require.Equal(t, expectedBalance, endBalance) + + // check timelock balance + endBalance, err = linkState.LinkToken.BalanceOf(&bind.CallOpts{Context: ctx}, timelockAddress) + require.NoError(t, err) + expectedBalance = big.NewInt(250) + require.Equal(t, expectedBalance, endBalance) +} + +// TestLinkTransfer tests the LinkTransfer changeset by sending LINK from a timelock contract to the deployer key. +func TestLinkTransfer(t *testing.T) { + t.Parallel() + ctx := context.Background() + + env := setupLinkTransferTestEnv(t) + chainSelector := env.AllChainSelectors()[0] + chain := env.Chains[chainSelector] + addrs, err := env.ExistingAddresses.AddressesForChain(chainSelector) + require.NoError(t, err) + require.Len(t, addrs, 6) + + mcmsState, err := changeset.MaybeLoadMCMSWithTimelockChainState(chain, addrs) + require.NoError(t, err) + linkState, err := changeset.MaybeLoadLinkTokenChainState(chain, addrs) + require.NoError(t, err) + timelockAddress := mcmsState.Timelock.Address() + + // Mint some funds + // grant minter permissions + tx, err := linkState.LinkToken.GrantMintRole(chain.DeployerKey, chain.DeployerKey.From) + require.NoError(t, err) + _, err = deployment.ConfirmIfNoError(chain, tx, err) + require.NoError(t, err) + + tx, err = linkState.LinkToken.Mint(chain.DeployerKey, chain.DeployerKey.From, big.NewInt(750)) + require.NoError(t, err) + _, err = deployment.ConfirmIfNoError(chain, tx, err) + require.NoError(t, err) + + timelocks := map[uint64]*proposalutils.TimelockExecutionContracts{ + chainSelector: { + Timelock: mcmsState.Timelock, + CallProxy: mcmsState.CallProxy, + }, + } + + // Apply the changeset + _, err = changeset.ApplyChangesets(t, env, timelocks, []changeset.ChangesetApplication{ + // the changeset produces proposals, ApplyChangesets will sign & execute them. + // in practice, signing and executing are separated processes. + { + Changeset: changeset.WrapChangeSet(example.LinkTransfer), + Config: &example.LinkTransferConfig{ + From: chain.DeployerKey.From, + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: { + { + To: timelockAddress, + Value: big.NewInt(500), + }, + }, + }, + // No MCMSConfig here means we'll execute the txs directly. + }, + }, + }) + require.NoError(t, err) + + // Check new balances + endBalance, err := linkState.LinkToken.BalanceOf(&bind.CallOpts{Context: ctx}, chain.DeployerKey.From) + require.NoError(t, err) + expectedBalance := big.NewInt(250) + require.Equal(t, expectedBalance, endBalance) + + // check timelock balance + endBalance, err = linkState.LinkToken.BalanceOf(&bind.CallOpts{Context: ctx}, timelockAddress) + require.NoError(t, err) + expectedBalance = big.NewInt(500) + require.Equal(t, expectedBalance, endBalance) +} + +func TestValidate(t *testing.T) { + env := setupLinkTransferTestEnv(t) + chainSelector := env.AllChainSelectors()[0] + chain := env.Chains[chainSelector] + addrs, err := env.ExistingAddresses.AddressesForChain(chainSelector) + require.NoError(t, err) + require.Len(t, addrs, 6) + mcmsState, err := changeset.MaybeLoadMCMSWithTimelockChainState(chain, addrs) + require.NoError(t, err) + linkState, err := changeset.MaybeLoadLinkTokenChainState(chain, addrs) + require.NoError(t, err) + tx, err := linkState.LinkToken.GrantMintRole(chain.DeployerKey, chain.DeployerKey.From) + require.NoError(t, err) + _, err = deployment.ConfirmIfNoError(chain, tx, err) + require.NoError(t, err) + tx, err = linkState.LinkToken.Mint(chain.DeployerKey, chain.DeployerKey.From, big.NewInt(750)) + require.NoError(t, err) + _, err = deployment.ConfirmIfNoError(chain, tx, err) + + require.NoError(t, err) + tests := []struct { + name string + cfg example.LinkTransferConfig + errorMsg string + }{ + { + name: "valid config", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: {{To: mcmsState.Timelock.Address(), Value: big.NewInt(100)}}}, + From: chain.DeployerKey.From, + McmsConfig: &example.MCMSConfig{ + MinDelay: time.Hour, + }, + }, + }, + { + name: "valid non mcms config", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: {{To: mcmsState.Timelock.Address(), Value: big.NewInt(100)}}}, + From: chain.DeployerKey.From, + }, + }, + { + name: "insufficient funds", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: { + {To: chain.DeployerKey.From, Value: big.NewInt(100)}, + {To: chain.DeployerKey.From, Value: big.NewInt(500)}, + {To: chain.DeployerKey.From, Value: big.NewInt(1250)}, + }, + }, + From: mcmsState.Timelock.Address(), + McmsConfig: &example.MCMSConfig{ + MinDelay: time.Hour, + }, + }, + errorMsg: "sender does not have enough funds for transfers for chain selector 909606746561742123, required: 1850, available: 0", + }, + { + name: "invalid config: empty transfers", + cfg: example.LinkTransferConfig{Transfers: map[uint64][]example.TransferConfig{}}, + errorMsg: "transfers map must have at least one chainSel", + }, + { + name: "invalid chain selector", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + 1: {{To: common.Address{}, Value: big.NewInt(100)}}}, + }, + errorMsg: "invalid chain selector: unknown chain selector 1", + }, + { + name: "chain selector not found", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chain_selectors.ETHEREUM_TESTNET_GOERLI_ARBITRUM_1.Selector: {{To: common.Address{}, Value: big.NewInt(100)}}}, + }, + errorMsg: "chain with selector 6101244977088475029 not found", + }, + { + name: "empty transfer list", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: {}, + }, + }, + errorMsg: "transfers for chainSel 909606746561742123 must have at least one LinkTransfer", + }, + { + name: "empty value", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: { + {To: chain.DeployerKey.From, Value: nil}, + }, + }, + }, + errorMsg: "value for transfers must be set", + }, + { + name: "zero value", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: { + {To: chain.DeployerKey.From, Value: big.NewInt(0)}, + }, + }, + }, + errorMsg: "value for transfers must be non-zero", + }, + { + name: "negative value", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: { + {To: chain.DeployerKey.From, Value: big.NewInt(-5)}, + }, + }, + }, + errorMsg: "value for transfers must be positive", + }, + { + name: "non-evm-chain", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chain_selectors.APTOS_MAINNET.Selector: {{To: mcmsState.Timelock.Address(), Value: big.NewInt(100)}}}, + From: chain.DeployerKey.From, + }, + errorMsg: "chain selector 4741433654826277614 is not an EVM chain", + }, + { + name: "delay greater than max allowed", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: {{To: mcmsState.Timelock.Address(), Value: big.NewInt(100)}}}, + From: chain.DeployerKey.From, + McmsConfig: &example.MCMSConfig{ + MinDelay: time.Hour * 24 * 10, + }, + }, + errorMsg: "minDelay must be less than 7 days", + }, + { + name: "invalid config: transfer to address missing", + cfg: example.LinkTransferConfig{ + Transfers: map[uint64][]example.TransferConfig{ + chainSelector: {{To: common.Address{}, Value: big.NewInt(100)}}}, + }, + errorMsg: "'to' address for transfers must be set", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.cfg.Validate(env) + if tt.errorMsg != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tt.errorMsg) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/deployment/common/changeset/example/mint_link.go b/deployment/common/changeset/example/mint_link.go new file mode 100644 index 00000000000..dc50f8a1a27 --- /dev/null +++ b/deployment/common/changeset/example/mint_link.go @@ -0,0 +1,43 @@ +package example + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/common/changeset" +) + +type MintLinkConfig struct { + Amount *big.Int + ChainSelector uint64 + To common.Address +} + +var _ deployment.ChangeSet[*MintLinkConfig] = MintLink + +// MintLink mints LINK to the provided contract. +func MintLink(e deployment.Environment, cfg *MintLinkConfig) (deployment.ChangesetOutput, error) { + + chain := e.Chains[cfg.ChainSelector] + addresses, err := e.ExistingAddresses.AddressesForChain(cfg.ChainSelector) + if err != nil { + return deployment.ChangesetOutput{}, err + } + linkState, err := changeset.MaybeLoadLinkTokenChainState(chain, addresses) + if err != nil { + return deployment.ChangesetOutput{}, err + } + + tx, err := linkState.LinkToken.Mint(chain.DeployerKey, cfg.To, cfg.Amount) + if err != nil { + return deployment.ChangesetOutput{}, err + } + _, err = deployment.ConfirmIfNoError(chain, tx, err) + if err != nil { + return deployment.ChangesetOutput{}, err + } + return deployment.ChangesetOutput{}, nil + +} diff --git a/deployment/common/changeset/example/mint_link_test.go b/deployment/common/changeset/example/mint_link_test.go new file mode 100644 index 00000000000..1c60c3221de --- /dev/null +++ b/deployment/common/changeset/example/mint_link_test.go @@ -0,0 +1,58 @@ +package example_test + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/deployment/common/changeset" + "github.com/smartcontractkit/chainlink/deployment/common/changeset/example" +) + +// TestMintLink tests the MintLink changeset +func TestMintLink(t *testing.T) { + t.Parallel() + env := setupLinkTransferTestEnv(t) + ctx := env.GetContext() + chainSelector := env.AllChainSelectors()[0] + chain := env.Chains[chainSelector] + + addrs, err := env.ExistingAddresses.AddressesForChain(chainSelector) + require.NoError(t, err) + require.Len(t, addrs, 6) + + mcmsState, err := changeset.MaybeLoadMCMSWithTimelockChainState(chain, addrs) + require.NoError(t, err) + linkState, err := changeset.MaybeLoadLinkTokenChainState(chain, addrs) + require.NoError(t, err) + + _, err = changeset.ApplyChangesets(t, env, nil, []changeset.ChangesetApplication{ + { + Changeset: changeset.WrapChangeSet(example.AddMintersBurnersLink), + Config: &example.AddMintersBurnersLinkConfig{ + ChainSelector: chainSelector, + Minters: []common.Address{chain.DeployerKey.From}, + }, + }, + }) + require.NoError(t, err) + + timelockAddress := mcmsState.Timelock.Address() + + // Mint some funds + _, err = example.MintLink(env, &example.MintLinkConfig{ + ChainSelector: chainSelector, + To: timelockAddress, + Amount: big.NewInt(7568), + }) + require.NoError(t, err) + + // check timelock balance + endBalance, err := linkState.LinkToken.BalanceOf(&bind.CallOpts{Context: ctx}, timelockAddress) + require.NoError(t, err) + expectedBalance := big.NewInt(7568) + require.Equal(t, expectedBalance, endBalance) +} diff --git a/deployment/common/changeset/internal/mcms_test.go b/deployment/common/changeset/internal/mcms_test.go index 8446aab4bfe..ff013717d30 100644 --- a/deployment/common/changeset/internal/mcms_test.go +++ b/deployment/common/changeset/internal/mcms_test.go @@ -40,7 +40,7 @@ func TestDeployMCMSWithTimelockContracts(t *testing.T) { addresses, err := ab.AddressesForChain(chainsel.TEST_90000001.Selector) require.NoError(t, err) require.Len(t, addresses, 5) - mcmsState, err := changeset.MaybeLoadMCMSWithTimelockState(chains[chainsel.TEST_90000001.Selector], addresses) + mcmsState, err := changeset.MaybeLoadMCMSWithTimelockChainState(chains[chainsel.TEST_90000001.Selector], addresses) require.NoError(t, err) v, err := mcmsState.GenerateMCMSWithTimelockView() b, err := json.MarshalIndent(v, "", " ") diff --git a/deployment/common/changeset/state.go b/deployment/common/changeset/state.go index c45fe6ba9b5..0db34abad71 100644 --- a/deployment/common/changeset/state.go +++ b/deployment/common/changeset/state.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + owner_helpers "github.com/smartcontractkit/ccip-owner-contracts/pkg/gethwrappers" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/common/proposalutils" @@ -22,17 +23,6 @@ type MCMSWithTimelockState struct { *proposalutils.MCMSWithTimelockContracts } -func MaybeLoadMCMSWithTimelockState(chain deployment.Chain, addresses map[string]deployment.TypeAndVersion) (*MCMSWithTimelockState, error) { - contracts, err := proposalutils.MaybeLoadMCMSWithTimelockContracts(chain, addresses) - if err != nil { - return nil, err - } - - return &MCMSWithTimelockState{ - MCMSWithTimelockContracts: contracts, - }, nil -} - func (state MCMSWithTimelockState) GenerateMCMSWithTimelockView() (v1_0.MCMSWithTimelockView, error) { if err := state.Validate(); err != nil { return v1_0.MCMSWithTimelockView{}, err @@ -66,6 +56,91 @@ func (state MCMSWithTimelockState) GenerateMCMSWithTimelockView() (v1_0.MCMSWith }, nil } +// MaybeLoadMCMSWithTimelockState loads the MCMSWithTimelockState state for each chain in the given environment. +func MaybeLoadMCMSWithTimelockState(env deployment.Environment, chainSelectors []uint64) (map[uint64]*MCMSWithTimelockState, error) { + result := map[uint64]*MCMSWithTimelockState{} + for _, chainSelector := range chainSelectors { + chain, ok := env.Chains[chainSelector] + if !ok { + return nil, fmt.Errorf("chain %d not found", chainSelector) + } + addressesChain, err := env.ExistingAddresses.AddressesForChain(chainSelector) + if err != nil { + return nil, err + } + state, err := MaybeLoadMCMSWithTimelockChainState(chain, addressesChain) + if err != nil { + return nil, err + } + result[chainSelector] = state + } + return result, nil +} + +// MaybeLoadMCMSWithTimelockChainState looks for the addresses corresponding to +// contracts deployed with DeployMCMSWithTimelock and loads them into a +// MCMSWithTimelockState struct. If none of the contracts are found, the state struct will be nil. +// An error indicates: +// - Found but was unable to load a contract +// - It only found part of the bundle of contracts +// - If found more than one instance of a contract (we expect one bundle in the given addresses) +func MaybeLoadMCMSWithTimelockChainState(chain deployment.Chain, addresses map[string]deployment.TypeAndVersion) (*MCMSWithTimelockState, error) { + state := MCMSWithTimelockState{ + MCMSWithTimelockContracts: &proposalutils.MCMSWithTimelockContracts{}, + } + // We expect one of each contract on the chain. + timelock := deployment.NewTypeAndVersion(types.RBACTimelock, deployment.Version1_0_0) + callProxy := deployment.NewTypeAndVersion(types.CallProxy, deployment.Version1_0_0) + proposer := deployment.NewTypeAndVersion(types.ProposerManyChainMultisig, deployment.Version1_0_0) + canceller := deployment.NewTypeAndVersion(types.CancellerManyChainMultisig, deployment.Version1_0_0) + bypasser := deployment.NewTypeAndVersion(types.BypasserManyChainMultisig, deployment.Version1_0_0) + + // Ensure we either have the bundle or not. + _, err := deployment.AddressesContainBundle(addresses, + map[deployment.TypeAndVersion]struct{}{ + timelock: {}, proposer: {}, canceller: {}, bypasser: {}, callProxy: {}, + }) + if err != nil { + return nil, fmt.Errorf("unable to check MCMS contracts on chain %s error: %w", chain.Name(), err) + } + + for address, tvStr := range addresses { + switch tvStr { + case timelock: + tl, err := owner_helpers.NewRBACTimelock(common.HexToAddress(address), chain.Client) + if err != nil { + return nil, err + } + state.Timelock = tl + case callProxy: + cp, err := owner_helpers.NewCallProxy(common.HexToAddress(address), chain.Client) + if err != nil { + return nil, err + } + state.CallProxy = cp + case proposer: + mcms, err := owner_helpers.NewManyChainMultiSig(common.HexToAddress(address), chain.Client) + if err != nil { + return nil, err + } + state.ProposerMcm = mcms + case bypasser: + mcms, err := owner_helpers.NewManyChainMultiSig(common.HexToAddress(address), chain.Client) + if err != nil { + return nil, err + } + state.BypasserMcm = mcms + case canceller: + mcms, err := owner_helpers.NewManyChainMultiSig(common.HexToAddress(address), chain.Client) + if err != nil { + return nil, err + } + state.CancellerMcm = mcms + } + } + return &state, nil +} + type LinkTokenState struct { LinkToken *link_token.LinkToken } @@ -77,7 +152,28 @@ func (s LinkTokenState) GenerateLinkView() (v1_0.LinkTokenView, error) { return v1_0.GenerateLinkTokenView(s.LinkToken) } -func MaybeLoadLinkTokenState(chain deployment.Chain, addresses map[string]deployment.TypeAndVersion) (*LinkTokenState, error) { +// MaybeLoadLinkTokenState loads the LinkTokenState state for each chain in the given environment. +func MaybeLoadLinkTokenState(env deployment.Environment, chainSelectors []uint64) (map[uint64]*LinkTokenState, error) { + result := map[uint64]*LinkTokenState{} + for _, chainSelector := range chainSelectors { + chain, ok := env.Chains[chainSelector] + if !ok { + return nil, fmt.Errorf("chain %d not found", chainSelector) + } + addressesChain, err := env.ExistingAddresses.AddressesForChain(chainSelector) + if err != nil { + return nil, err + } + state, err := MaybeLoadLinkTokenChainState(chain, addressesChain) + if err != nil { + return nil, err + } + result[chainSelector] = state + } + return result, nil +} + +func MaybeLoadLinkTokenChainState(chain deployment.Chain, addresses map[string]deployment.TypeAndVersion) (*LinkTokenState, error) { state := LinkTokenState{} linkToken := deployment.NewTypeAndVersion(types.LinkToken, deployment.Version1_0_0) // Perhaps revisit if we have a use case for multiple. diff --git a/deployment/common/changeset/transfer_to_mcms_with_timelock_test.go b/deployment/common/changeset/transfer_to_mcms_with_timelock_test.go index 40cef99a54f..7ba11596a2d 100644 --- a/deployment/common/changeset/transfer_to_mcms_with_timelock_test.go +++ b/deployment/common/changeset/transfer_to_mcms_with_timelock_test.go @@ -34,9 +34,9 @@ func TestTransferToMCMSWithTimelock(t *testing.T) { require.NoError(t, err) addrs, err := e.ExistingAddresses.AddressesForChain(chain1) require.NoError(t, err) - state, err := MaybeLoadMCMSWithTimelockState(e.Chains[chain1], addrs) + state, err := MaybeLoadMCMSWithTimelockChainState(e.Chains[chain1], addrs) require.NoError(t, err) - link, err := MaybeLoadLinkTokenState(e.Chains[chain1], addrs) + link, err := MaybeLoadLinkTokenChainState(e.Chains[chain1], addrs) require.NoError(t, err) e, err = ApplyChangesets(t, e, map[uint64]*proposalutils.TimelockExecutionContracts{ chain1: { @@ -56,7 +56,7 @@ func TestTransferToMCMSWithTimelock(t *testing.T) { }) require.NoError(t, err) // We expect now that the link token is owned by the MCMS timelock. - link, err = MaybeLoadLinkTokenState(e.Chains[chain1], addrs) + link, err = MaybeLoadLinkTokenChainState(e.Chains[chain1], addrs) require.NoError(t, err) o, err := link.LinkToken.Owner(nil) require.NoError(t, err) diff --git a/deployment/common/proposalutils/propose.go b/deployment/common/proposalutils/propose.go index feaee69940e..32a5bcdfda2 100644 --- a/deployment/common/proposalutils/propose.go +++ b/deployment/common/proposalutils/propose.go @@ -15,7 +15,8 @@ const ( DefaultValidUntil = 72 * time.Hour ) -func buildProposalMetadata( + +func BuildProposalMetadata( chainSelectors []uint64, proposerMcmsesPerChain map[uint64]*gethwrappers.ManyChainMultiSig, ) (map[mcms.ChainIdentifier]mcms.ChainMetadata, error) { @@ -56,7 +57,7 @@ func BuildProposalFromBatches( chains.Add(uint64(op.ChainIdentifier)) } - mcmsMd, err := buildProposalMetadata(chains.ToSlice(), proposerMcmsesPerChain) + mcmsMd, err := BuildProposalMetadata(chains.ToSlice(), proposerMcmsesPerChain) if err != nil { return nil, err } diff --git a/deployment/environment.go b/deployment/environment.go index c9de89b8c0c..0823404da2d 100644 --- a/deployment/environment.go +++ b/deployment/environment.go @@ -181,7 +181,7 @@ func MaybeDataErr(err error) error { var d rpc.DataError ok := errors.As(err, &d) if ok { - return d + return fmt.Errorf("%s: %v", d.Error(), d.ErrorData()) } return err } diff --git a/deployment/keystone/changeset/accept_ownership_test.go b/deployment/keystone/changeset/accept_ownership_test.go index 9e9d29e563a..d949e63c7aa 100644 --- a/deployment/keystone/changeset/accept_ownership_test.go +++ b/deployment/keystone/changeset/accept_ownership_test.go @@ -51,7 +51,7 @@ func TestAcceptAllOwnership(t *testing.T) { require.NoError(t, err) addrs, err := env.ExistingAddresses.AddressesForChain(registrySel) require.NoError(t, err) - timelock, err := commonchangeset.MaybeLoadMCMSWithTimelockState(env.Chains[registrySel], addrs) + timelock, err := commonchangeset.MaybeLoadMCMSWithTimelockChainState(env.Chains[registrySel], addrs) require.NoError(t, err) _, err = commonchangeset.ApplyChangesets(t, env, map[uint64]*proposalutils.TimelockExecutionContracts{ diff --git a/deployment/keystone/state.go b/deployment/keystone/state.go index cbf449c7f31..0ac7cdc89ed 100644 --- a/deployment/keystone/state.go +++ b/deployment/keystone/state.go @@ -78,7 +78,7 @@ func GetContractSets(lggr logger.Logger, req *GetContractSetsRequest) (*GetContr func loadContractSet(lggr logger.Logger, chain deployment.Chain, addresses map[string]deployment.TypeAndVersion) (*ContractSet, error) { var out ContractSet - mcmsWithTimelock, err := commonchangeset.MaybeLoadMCMSWithTimelockState(chain, addresses) + mcmsWithTimelock, err := commonchangeset.MaybeLoadMCMSWithTimelockChainState(chain, addresses) if err != nil { return nil, fmt.Errorf("failed to load mcms contract: %w", err) } From 8f1b956efe05aa9ca819d755d6c1386740bd5533 Mon Sep 17 00:00:00 2001 From: Austin <107539019+0xAustinWang@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:57:09 +0800 Subject: [PATCH 10/11] Initialization for Crib (#15501) * wip * test crib integration flow * build failures * changes from ani's comments * some changes * revert isempty check * go lint and compare integers * fix types in test * check for error if there's already a feeds manager * check if job distributors exist first * wip changes * add retries to consistently failing JD cals * update retries with static duration * remove print statements * fix compile error * remove unnecessary mcms changes * build issues * passing deployment * clean up code comments * gomodtidy * lint * formatting strings * use the common changeset utilities when deploying home chain as well * compare nodes job distributors public key against the one we expect * move state reader under deployment module * add RPC type with internal and external rpcs --------- Co-authored-by: Radek Scheibinger --- deployment/address_book.go | 4 +- deployment/environment/crib/ccip_deployer.go | 136 ++++++++++++++++++ deployment/environment/crib/data.go | 81 +++++++++++ deployment/environment/crib/env.go | 45 ++++++ deployment/environment/crib/env_test.go | 18 +++ .../ccip-v2-scripts-address-book.json | 1 + .../ccip-v2-scripts-chains-details.json | 24 ++++ .../ccip-v2-scripts-nodes-details.json | 1 + deployment/environment/crib/types.go | 39 +++++ deployment/environment/devenv/don.go | 88 ++++++++---- .../environment/web/sdk/client/client.go | 16 ++- integration-tests/load/go.mod | 1 + integration-tests/testconfig/ccip/config.go | 22 ++- 13 files changed, 435 insertions(+), 41 deletions(-) create mode 100644 deployment/environment/crib/ccip_deployer.go create mode 100644 deployment/environment/crib/data.go create mode 100644 deployment/environment/crib/env.go create mode 100644 deployment/environment/crib/env_test.go create mode 100644 deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-address-book.json create mode 100644 deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-chains-details.json create mode 100644 deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-nodes-details.json create mode 100644 deployment/environment/crib/types.go diff --git a/deployment/address_book.go b/deployment/address_book.go index 6f605013011..3ce0332a4c3 100644 --- a/deployment/address_book.go +++ b/deployment/address_book.go @@ -89,8 +89,10 @@ type AddressBook interface { Remove(ab AddressBook) error } +type AddressesByChain map[uint64]map[string]TypeAndVersion + type AddressBookMap struct { - addressesByChain map[uint64]map[string]TypeAndVersion + addressesByChain AddressesByChain mtx sync.RWMutex } diff --git a/deployment/environment/crib/ccip_deployer.go b/deployment/environment/crib/ccip_deployer.go new file mode 100644 index 00000000000..aea7ad0cb8f --- /dev/null +++ b/deployment/environment/crib/ccip_deployer.go @@ -0,0 +1,136 @@ +package crib + +import ( + "context" + "errors" + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/smartcontractkit/ccip-owner-contracts/pkg/config" + commonchangeset "github.com/smartcontractkit/chainlink/deployment/common/changeset" + commontypes "github.com/smartcontractkit/chainlink/deployment/common/types" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" + "github.com/smartcontractkit/chainlink/v2/core/services/relay" + "math/big" + + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" + "github.com/smartcontractkit/chainlink/v2/core/logger" +) + +// DeployHomeChainContracts deploys the home chain contracts so that the chainlink nodes can be started with the CR address in Capabilities.ExternalRegistry +// DeployHomeChainContracts is to 1. Set up crib with chains and chainlink nodes ( cap reg is not known yet so not setting the config with capreg address) +// Call DeployHomeChain changeset with nodeinfo ( the peer id and all) +func DeployHomeChainContracts(ctx context.Context, lggr logger.Logger, envConfig devenv.EnvironmentConfig, homeChainSel uint64, feedChainSel uint64) (deployment.CapabilityRegistryConfig, deployment.AddressBook, error) { + e, _, err := devenv.NewEnvironment(func() context.Context { return ctx }, lggr, envConfig) + if err != nil { + return deployment.CapabilityRegistryConfig{}, nil, err + } + if e == nil { + return deployment.CapabilityRegistryConfig{}, nil, errors.New("environment is nil") + } + + nodes, err := deployment.NodeInfo(e.NodeIDs, e.Offchain) + if err != nil { + return deployment.CapabilityRegistryConfig{}, e.ExistingAddresses, fmt.Errorf("failed to get node info from env: %w", err) + } + p2pIds := nodes.NonBootstraps().PeerIDs() + *e, err = commonchangeset.ApplyChangesets(nil, *e, nil, []commonchangeset.ChangesetApplication{ + { + Changeset: commonchangeset.WrapChangeSet(changeset.DeployHomeChain), + Config: changeset.DeployHomeChainConfig{ + HomeChainSel: homeChainSel, + RMNStaticConfig: changeset.NewTestRMNStaticConfig(), + RMNDynamicConfig: changeset.NewTestRMNDynamicConfig(), + NodeOperators: changeset.NewTestNodeOperator(e.Chains[homeChainSel].DeployerKey.From), + NodeP2PIDsPerNodeOpAdmin: map[string][][32]byte{ + "NodeOperator": p2pIds, + }, + }, + }, + }) + + state, err := changeset.LoadOnchainState(*e) + if err != nil { + return deployment.CapabilityRegistryConfig{}, e.ExistingAddresses, fmt.Errorf("failed to load on chain state: %w", err) + } + capRegAddr := state.Chains[homeChainSel].CapabilityRegistry.Address() + if capRegAddr == common.HexToAddress("0x") { + return deployment.CapabilityRegistryConfig{}, e.ExistingAddresses, fmt.Errorf("cap Reg address not found: %w", err) + } + capRegConfig := deployment.CapabilityRegistryConfig{ + EVMChainID: homeChainSel, + Contract: state.Chains[homeChainSel].CapabilityRegistry.Address(), + NetworkType: relay.NetworkEVM, + } + return capRegConfig, e.ExistingAddresses, nil +} + +func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig devenv.EnvironmentConfig, homeChainSel, feedChainSel uint64, ab deployment.AddressBook) (DeployCCIPOutput, error) { + e, _, err := devenv.NewEnvironment(func() context.Context { return ctx }, lggr, envConfig) + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to initiate new environment: %w", err) + } + e.ExistingAddresses = ab + allChainIds := e.AllChainSelectors() + cfg := make(map[uint64]commontypes.MCMSWithTimelockConfig) + for _, chain := range e.AllChainSelectors() { + mcmsConfig, err := config.NewConfig(1, []common.Address{e.Chains[chain].DeployerKey.From}, []config.Config{}) + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to create mcms config: %w", err) + } + cfg[chain] = commontypes.MCMSWithTimelockConfig{ + Canceller: *mcmsConfig, + Bypasser: *mcmsConfig, + Proposer: *mcmsConfig, + TimelockMinDelay: big.NewInt(0), + } + } + + // This will not apply any proposals because we pass nil to testing. + // However, setup is ok because we only need to deploy the contracts and distribute job specs + *e, err = commonchangeset.ApplyChangesets(nil, *e, nil, []commonchangeset.ChangesetApplication{ + { + Changeset: commonchangeset.WrapChangeSet(commonchangeset.DeployLinkToken), + Config: allChainIds, + }, + { + Changeset: commonchangeset.WrapChangeSet(changeset.DeployPrerequisites), + Config: changeset.DeployPrerequisiteConfig{ + ChainSelectors: allChainIds, + }, + }, + { + Changeset: commonchangeset.WrapChangeSet(commonchangeset.DeployMCMSWithTimelock), + Config: cfg, + }, + { + Changeset: commonchangeset.WrapChangeSet(changeset.DeployChainContracts), + Config: changeset.DeployChainContractsConfig{ + ChainSelectors: allChainIds, + HomeChainSelector: homeChainSel, + }, + }, + { + Changeset: commonchangeset.WrapChangeSet(changeset.CCIPCapabilityJobspec), + Config: struct{}{}, + }, + }) + state, err := changeset.LoadOnchainState(*e) + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to load onchain state: %w", err) + } + // Add all lanes + err = changeset.AddLanesForAll(*e, state) + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to add lanes: %w", err) + } + + addresses, err := e.ExistingAddresses.Addresses() + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to get convert address book to address book map: %w", err) + } + return DeployCCIPOutput{ + AddressBook: *deployment.NewMemoryAddressBookFromMap(addresses), + NodeIDs: e.NodeIDs, + }, err +} diff --git a/deployment/environment/crib/data.go b/deployment/environment/crib/data.go new file mode 100644 index 00000000000..b9197691613 --- /dev/null +++ b/deployment/environment/crib/data.go @@ -0,0 +1,81 @@ +package crib + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" +) + +type OutputReader struct { + outputDir string +} + +func NewOutputReader(outputDir string) *OutputReader { + return &OutputReader{outputDir: outputDir} +} + +func (r *OutputReader) ReadNodesDetails() NodesDetails { + byteValue := r.readFile(NodesDetailsFileName) + + var result NodesDetails + + // Unmarshal the JSON into the map + err := json.Unmarshal(byteValue, &result) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + panic(err) + } + + return result +} + +func (r *OutputReader) ReadChainConfigs() []devenv.ChainConfig { + byteValue := r.readFile(ChainsConfigsFileName) + + var result []devenv.ChainConfig + + // Unmarshal the JSON into the map + err := json.Unmarshal(byteValue, &result) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + panic(err) + } + + return result +} + +func (r *OutputReader) ReadAddressBook() *deployment.AddressBookMap { + byteValue := r.readFile(AddressBookFileName) + + var result map[uint64]map[string]deployment.TypeAndVersion + + // Unmarshal the JSON into the map + err := json.Unmarshal(byteValue, &result) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + panic(err) + } + + return deployment.NewMemoryAddressBookFromMap(result) +} + +func (r *OutputReader) readFile(fileName string) []byte { + file, err := os.Open(fmt.Sprintf("%s/%s", r.outputDir, fileName)) + if err != nil { + fmt.Println("Error opening file:", err) + panic(err) + } + defer file.Close() + + // Read the file's content into a byte slice + byteValue, err := io.ReadAll(file) + if err != nil { + fmt.Println("Error reading file:", err) + panic(err) + } + return byteValue +} diff --git a/deployment/environment/crib/env.go b/deployment/environment/crib/env.go new file mode 100644 index 00000000000..3af1acaf754 --- /dev/null +++ b/deployment/environment/crib/env.go @@ -0,0 +1,45 @@ +package crib + +const ( + AddressBookFileName = "ccip-v2-scripts-address-book.json" + NodesDetailsFileName = "ccip-v2-scripts-nodes-details.json" + ChainsConfigsFileName = "ccip-v2-scripts-chains-details.json" +) + +type CRIBEnv struct { + envStateDir string +} + +func NewDevspaceEnvFromStateDir(envStateDir string) CRIBEnv { + return CRIBEnv{ + envStateDir: envStateDir, + } +} + +func (c CRIBEnv) GetConfig() DeployOutput { + reader := NewOutputReader(c.envStateDir) + nodesDetails := reader.ReadNodesDetails() + chainConfigs := reader.ReadChainConfigs() + return DeployOutput{ + AddressBook: reader.ReadAddressBook(), + NodeIDs: nodesDetails.NodeIDs, + Chains: chainConfigs, + } +} + +type RPC struct { + External *string + Internal *string +} + +type ChainConfig struct { + ChainID uint64 // chain id as per EIP-155, mainly applicable for EVM chains + ChainName string // name of the chain populated from chainselector repo + ChainType string // should denote the chain family. Acceptable values are EVM, COSMOS, SOLANA, STARKNET, APTOS etc + WSRPCs []RPC // websocket rpcs to connect to the chain + HTTPRPCs []RPC // http rpcs to connect to the chain +} + +type NodesDetails struct { + NodeIDs []string +} diff --git a/deployment/environment/crib/env_test.go b/deployment/environment/crib/env_test.go new file mode 100644 index 00000000000..262a2540923 --- /dev/null +++ b/deployment/environment/crib/env_test.go @@ -0,0 +1,18 @@ +package crib + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestShouldProvideEnvironmentConfig(t *testing.T) { + t.Parallel() + env := NewDevspaceEnvFromStateDir("testdata/lanes-deployed-state") + config := env.GetConfig() + require.NotNil(t, config) + assert.NotEmpty(t, config.NodeIDs) + assert.NotNil(t, config.AddressBook) + assert.NotEmpty(t, config.Chains) +} diff --git a/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-address-book.json b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-address-book.json new file mode 100644 index 00000000000..e4b2672cb5f --- /dev/null +++ b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-address-book.json @@ -0,0 +1 @@ +{"12922642891491394802":{"0x05Aa229Aec102f78CE0E852A812a388F076Aa555":{"Type":"CancellerManyChainMultiSig","Version":"1.0.0"},"0x0D4ff719551E23185Aeb16FFbF2ABEbB90635942":{"Type":"TestRouter","Version":"1.2.0"},"0x0f5D1ef48f12b6f691401bfe88c2037c690a6afe":{"Type":"ProposerManyChainMultiSig","Version":"1.0.0"},"0x2dE080e97B0caE9825375D31f5D0eD5751fDf16D":{"Type":"CCIPReceiver","Version":"1.0.0"},"0x2fc631e4B3018258759C52AF169200213e84ABab":{"Type":"OnRamp","Version":"1.6.0-dev"},"0x5C7c905B505f0Cf40Ab6600d05e677F717916F6B":{"Type":"Router","Version":"1.2.0"},"0x63cf2Cd54fE91e3545D1379abf5bfd194545259d":{"Type":"OffRamp","Version":"1.6.0-dev"},"0x712516e61C8B383dF4A63CFe83d7701Bce54B03e":{"Type":"LinkToken","Version":"1.0.0"},"0x71C95911E9a5D330f4D621842EC243EE1343292e":{"Type":"PriceFeed","Version":"1.0.0"},"0x73eccD6288e117cAcA738BDAD4FEC51312166C1A":{"Type":"RMNRemote","Version":"1.6.0-dev"},"0x8464135c8F25Da09e49BC8782676a84730C318bC":{"Type":"PriceFeed","Version":"1.0.0"},"0x85C5Dd61585773423e378146D4bEC6f8D149E248":{"Type":"TokenAdminRegistry","Version":"1.5.0"},"0x948B3c65b89DF0B4894ABE91E6D02FE579834F8F":{"Type":"WETH9","Version":"1.0.0"},"0xAfe1b5bdEbD4ae65AF2024738bf0735fbb65d44b":{"Type":"FeeQuoter","Version":"1.6.0-dev"},"0xC6bA8C3233eCF65B761049ef63466945c362EdD2":{"Type":"BypasserManyChainMultiSig","Version":"1.0.0"},"0xbCF26943C0197d2eE0E5D05c716Be60cc2761508":{"Type":"AdminManyChainMultiSig","Version":"1.0.0"},"0xcA03Dc4665A8C3603cb4Fd5Ce71Af9649dC00d44":{"Type":"RBACTimelock","Version":"1.0.0"},"0xe6b98F104c1BEf218F3893ADab4160Dc73Eb8367":{"Type":"ARMProxy","Version":"1.0.0"},"0xfbAb4aa40C202E4e80390171E82379824f7372dd":{"Type":"NonceManager","Version":"1.6.0-dev"}},"3379446385462418246":{"0x09635F643e140090A9A8Dcd712eD6285858ceBef":{"Type":"RMNRemote","Version":"1.6.0-dev"},"0x0B306BF915C4d645ff596e518fAf3F9669b97016":{"Type":"LinkToken","Version":"1.0.0"},"0x1613beB3B2C4f22Ee086B2b38C1476A3cE7f78E8":{"Type":"OnRamp","Version":"1.6.0-dev"},"0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6":{"Type":"CCIPHome","Version":"1.6.0-dev"},"0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44":{"Type":"ProposerManyChainMultiSig","Version":"1.0.0"},"0x3Aa5ebB10DC797CAC828524e59A333d0A371443c":{"Type":"BypasserManyChainMultiSig","Version":"1.0.0"},"0x4A679253410272dd5232B3Ff7cF5dbB88f295319":{"Type":"RBACTimelock","Version":"1.0.0"},"0x59b670e9fA9D0A427751Af201D676719a970857b":{"Type":"CancellerManyChainMultiSig","Version":"1.0.0"},"0x67d269191c92Caf3cD7723F116c85e6E9bf55933":{"Type":"ARMProxy","Version":"1.0.0"},"0x7a2088a1bFc9d81c55368AE168C2C02570cB814F":{"Type":"CCIPReceiver","Version":"1.0.0"},"0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB":{"Type":"TokenAdminRegistry","Version":"1.5.0"},"0x851356ae760d987E095750cCeb3bC6014560891C":{"Type":"OffRamp","Version":"1.6.0-dev"},"0x8A791620dd6260079BF849Dc5567aDC3F2FdC318":{"Type":"RMNHome","Version":"1.6.0-dev"},"0x9A676e781A523b5d0C0e43731313A708CB607508":{"Type":"WETH9","Version":"1.0.0"},"0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE":{"Type":"AdminManyChainMultiSig","Version":"1.0.0"},"0x9E545E3C0baAB3E08CdfD552C960A1050f373042":{"Type":"NonceManager","Version":"1.6.0-dev"},"0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E":{"Type":"Router","Version":"1.2.0"},"0xa513E6E4b8f2a923D98304ec87F64353C4D5C853":{"Type":"CapabilitiesRegistry","Version":"1.0.0"},"0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9":{"Type":"FeeQuoter","Version":"1.6.0-dev"},"0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690":{"Type":"TestRouter","Version":"1.2.0"}}} diff --git a/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-chains-details.json b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-chains-details.json new file mode 100644 index 00000000000..f93ea4ce231 --- /dev/null +++ b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-chains-details.json @@ -0,0 +1,24 @@ +[ + { + "ChainID": 1337, + "ChainName": "alpha", + "ChainType": "EVM", + "WSRPCs": [ + "wss://crib-local-geth-1337-ws.local:443" + ], + "HTTPRPCs": [ + "https://crib-local-geth-1337-ws.local:443" + ] + }, + { + "ChainID": 2337, + "ChainName": "alpha", + "ChainType": "EVM", + "WSRPCs": [ + "wss://crib-local-geth-2337-ws.local:443" + ], + "HTTPRPCs": [ + "https://crib-local-geth-2337-ws.local:443" + ] + } +] diff --git a/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-nodes-details.json b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-nodes-details.json new file mode 100644 index 00000000000..477ae0527b1 --- /dev/null +++ b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-nodes-details.json @@ -0,0 +1 @@ +{"NodeIDs":["node_2URuou3RXmtZu5gLQX8qd","node_m9TTQbUxBx3WjDEjmpVDL","node_4FiKVPtuQjCTvHnS7QpES","node_A4VTgecDwMoG2YYicyjuG","node_jQFpzXDadzaADq147nThS"]} diff --git a/deployment/environment/crib/types.go b/deployment/environment/crib/types.go new file mode 100644 index 00000000000..d19c8424443 --- /dev/null +++ b/deployment/environment/crib/types.go @@ -0,0 +1,39 @@ +package crib + +import ( + "context" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" +) + +const ( + CRIB_ENV_NAME = "Crib Environment" +) + +type DeployOutput struct { + NodeIDs []string + Chains []devenv.ChainConfig // chain selector -> Chain Config + AddressBook deployment.AddressBook // Addresses of all contracts +} + +type DeployCCIPOutput struct { + AddressBook deployment.AddressBookMap + NodeIDs []string +} + +func NewDeployEnvironmentFromCribOutput(lggr logger.Logger, output DeployOutput) (*deployment.Environment, error) { + chains, err := devenv.NewChains(lggr, output.Chains) + if err != nil { + return nil, err + } + return deployment.NewEnvironment( + CRIB_ENV_NAME, + lggr, + output.AddressBook, + chains, + output.NodeIDs, + nil, // todo: populate the offchain client using output.DON + func() context.Context { return context.Background() }, deployment.XXXGenerateTestOCRSecrets(), + ), nil +} diff --git a/deployment/environment/devenv/don.go b/deployment/environment/devenv/don.go index 05a3d5bea08..76f6ee92b68 100644 --- a/deployment/environment/devenv/don.go +++ b/deployment/environment/devenv/don.go @@ -2,7 +2,9 @@ package devenv import ( "context" + "errors" "fmt" + chainsel "github.com/smartcontractkit/chain-selectors" "strconv" "strings" "time" @@ -10,8 +12,6 @@ import ( "github.com/hashicorp/go-multierror" "github.com/rs/zerolog" "github.com/sethvargo/go-retry" - chainsel "github.com/smartcontractkit/chain-selectors" - nodev1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" clclient "github.com/smartcontractkit/chainlink/deployment/environment/nodeclient" "github.com/smartcontractkit/chainlink/deployment/environment/web/sdk/client" @@ -185,7 +185,7 @@ type JDChainConfigInput struct { // It expects bootstrap nodes to have label with key "type" and value as "bootstrap". // It fetches the account address, peer id, and OCR2 key bundle id and creates the JobDistributorChainConfig. func (n *Node) CreateCCIPOCRSupportedChains(ctx context.Context, chains []JDChainConfigInput, jd JobDistributor) error { - for i, chain := range chains { + for _, chain := range chains { chainId := strconv.FormatUint(chain.ChainID, 10) var account string switch chain.ChainType { @@ -239,35 +239,51 @@ func (n *Node) CreateCCIPOCRSupportedChains(ctx context.Context, chains []JDChai break } } - // JD silently fails to update nodeChainConfig. Therefore, we fetch the node config and - // if it's not updated , throw an error - _, err = n.gqlClient.CreateJobDistributorChainConfig(ctx, client.JobDistributorChainConfigInput{ - JobDistributorID: n.JDId, - ChainID: chainId, - ChainType: chain.ChainType, - AccountAddr: account, - AdminAddr: n.adminAddr, - Ocr2Enabled: true, - Ocr2IsBootstrap: isBootstrap, - Ocr2Multiaddr: n.multiAddr, - Ocr2P2PPeerID: value(peerID), - Ocr2KeyBundleID: ocr2BundleId, - Ocr2Plugins: `{"commit":true,"execute":true,"median":false,"mercury":false}`, + + // retry twice with 5 seconds interval to create JobDistributorChainConfig + err = retry.Do(ctx, retry.WithMaxDuration(10*time.Second, retry.NewConstant(3*time.Second)), func(ctx context.Context) error { + // check the node chain config to see if this chain already exists + nodeChainConfigs, err := jd.ListNodeChainConfigs(context.Background(), &nodev1.ListNodeChainConfigsRequest{ + Filter: &nodev1.ListNodeChainConfigsRequest_Filter{ + NodeIds: []string{n.NodeId}, + }}) + if err != nil { + return retry.RetryableError(fmt.Errorf("failed to list node chain configs for node %s, retrying..: %w", n.Name, err)) + } + if nodeChainConfigs != nil { + for _, chainConfig := range nodeChainConfigs.ChainConfigs { + if chainConfig.Chain.Id == chainId { + return nil + } + } + } + + // JD silently fails to update nodeChainConfig. Therefore, we fetch the node config and + // if it's not updated , throw an error + _, err = n.gqlClient.CreateJobDistributorChainConfig(ctx, client.JobDistributorChainConfigInput{ + JobDistributorID: n.JDId, + ChainID: chainId, + ChainType: chain.ChainType, + AccountAddr: account, + AdminAddr: n.adminAddr, + Ocr2Enabled: true, + Ocr2IsBootstrap: isBootstrap, + Ocr2Multiaddr: n.multiAddr, + Ocr2P2PPeerID: value(peerID), + Ocr2KeyBundleID: ocr2BundleId, + Ocr2Plugins: `{"commit":true,"execute":true,"median":false,"mercury":false}`, + }) + // todo: add a check if the chain config failed because of a duplicate in that case, should we update or return success? + if err != nil { + return fmt.Errorf("failed to create CCIPOCR2SupportedChains for node %s: %w", n.Name, err) + } + + return retry.RetryableError(errors.New("retrying CreateChainConfig in JD")) }) + if err != nil { return fmt.Errorf("failed to create CCIPOCR2SupportedChains for node %s: %w", n.Name, err) } - // query the node chain config to check if it's created - nodeChainConfigs, err := jd.ListNodeChainConfigs(context.Background(), &nodev1.ListNodeChainConfigsRequest{ - Filter: &nodev1.ListNodeChainConfigsRequest_Filter{ - NodeIds: []string{n.NodeId}, - }}) - if err != nil { - return fmt.Errorf("failed to list node chain configs for node %s: %w", n.Name, err) - } - if nodeChainConfigs == nil || len(nodeChainConfigs.ChainConfigs) < i+1 { - return fmt.Errorf("failed to create chain config for node %s", n.Name) - } } return nil } @@ -377,6 +393,17 @@ func (n *Node) CreateJobDistributor(ctx context.Context, jd JobDistributor) (str return "", err } // create the job distributor in the node with the csa key + resp, err := n.gqlClient.ListJobDistributors(ctx) + if err != nil { + return "", fmt.Errorf("could not list job distrubutors: %w", err) + } + if len(resp.FeedsManagers.Results) > 0 { + for _, fm := range resp.FeedsManagers.Results { + if fm.GetPublicKey() == csaKey { + return fm.GetId(), nil + } + } + } return n.gqlClient.CreateJobDistributor(ctx, client.JobDistributorInput{ Name: "Job Distributor", Uri: jd.WSRPC, @@ -394,8 +421,9 @@ func (n *Node) SetUpAndLinkJobDistributor(ctx context.Context, jd JobDistributor } // now create the job distributor in the node id, err := n.CreateJobDistributor(ctx, jd) - if err != nil && !strings.Contains(err.Error(), "DuplicateFeedsManagerError") { - return err + if err != nil && + (!strings.Contains(err.Error(), "only a single feeds manager is supported") || !strings.Contains(err.Error(), "DuplicateFeedsManagerError")) { + return fmt.Errorf("failed to create job distributor in node %s: %w", n.Name, err) } // wait for the node to connect to the job distributor err = retry.Do(ctx, retry.WithMaxDuration(1*time.Minute, retry.NewFibonacci(1*time.Second)), func(ctx context.Context) error { diff --git a/deployment/environment/web/sdk/client/client.go b/deployment/environment/web/sdk/client/client.go index 5472591ef94..e0a56b9e642 100644 --- a/deployment/environment/web/sdk/client/client.go +++ b/deployment/environment/web/sdk/client/client.go @@ -4,10 +4,11 @@ import ( "context" "encoding/json" "fmt" + "github.com/Khan/genqlient/graphql" + "github.com/sethvargo/go-retry" "net/http" "strings" - - "github.com/Khan/genqlient/graphql" + "time" "github.com/smartcontractkit/chainlink/deployment/environment/web/sdk/client/doer" "github.com/smartcontractkit/chainlink/deployment/environment/web/sdk/internal/generated" @@ -60,8 +61,15 @@ func New(baseURI string, creds Credentials) (Client, error) { endpoints: ep, credentials: creds, } - - if err := c.login(); err != nil { + + err := retry.Do(context.Background(), retry.WithMaxDuration(10*time.Second, retry.NewFibonacci(2*time.Second)), func(ctx context.Context) error { + err := c.login() + if err != nil { + return retry.RetryableError(fmt.Errorf("retrying login to node: %w", err)) + } + return nil + }) + if err != nil { return nil, fmt.Errorf("failed to login to node: %w", err) } diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 3d240cccc9e..c94ea489c1b 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -398,6 +398,7 @@ require ( github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/segmentio/ksuid v1.0.4 // indirect github.com/sercand/kuberesolver/v5 v5.1.1 // indirect + github.com/sethvargo/go-retry v0.2.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shirou/gopsutil/v3 v3.24.3 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect diff --git a/integration-tests/testconfig/ccip/config.go b/integration-tests/testconfig/ccip/config.go index 72c81f05f47..70c850fd591 100644 --- a/integration-tests/testconfig/ccip/config.go +++ b/integration-tests/testconfig/ccip/config.go @@ -147,6 +147,9 @@ func (o *JDConfig) GetJDDBVersion() string { func (o *Config) Validate() error { var chainIds []int64 for _, net := range o.PrivateEthereumNetworks { + if net.EthereumChainConfig.ChainID < 0 { + return fmt.Errorf("negative chain ID found for network %d", net.EthereumChainConfig.ChainID) + } chainIds = append(chainIds, int64(net.EthereumChainConfig.ChainID)) } homeChainSelector, err := strconv.ParseUint(pointer.GetString(o.HomeChainSelector), 10, 64) @@ -189,14 +192,21 @@ func IsSelectorValid(selector uint64, chainIds []int64) (bool, error) { if err != nil { return false, err } - if chainId >= math.MaxInt64 { - return false, fmt.Errorf("chain id overflows int64: %d", chainId) - } - expId := int64(chainId) - for _, id := range chainIds { - if id == expId { + + for _, cID := range chainIds { + if isEqualUint64AndInt64(chainId, cID) { return true, nil } } return false, nil } + +func isEqualUint64AndInt64(u uint64, i int64) bool { + if i < 0 { + return false // uint64 cannot be equal to a negative int64 + } + if u > math.MaxInt64 { + return false // uint64 cannot be equal to an int64 if it exceeds the maximum int64 value + } + return u == uint64(i) +} From 6f42463b557a9c2193977e60c59f042f3ef28aa0 Mon Sep 17 00:00:00 2001 From: dimitris Date: Fri, 13 Dec 2024 12:07:32 +0200 Subject: [PATCH 11/11] RMN tests CI reliability and one new test scenario (#15677) * re-enable rmn tests with larger runners * add uncurse test case * fix assignment to nil map --- .github/e2e-tests.yml | 16 ++-- integration-tests/smoke/ccip/ccip_rmn_test.go | 94 +++++++++++++++---- 2 files changed, 82 insertions(+), 28 deletions(-) diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 1bf55a64418..93ddbb564b6 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -966,7 +966,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_TwoMessagesOnTwoLanesIncludingBatching$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -982,7 +982,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_MultipleMessagesOnOneLaneNoWaitForExec$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -998,7 +998,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_NotEnoughObservers$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -1014,7 +1014,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_DifferentSigners$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -1030,7 +1030,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_NotEnoughSigners$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -1046,7 +1046,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_DifferentRmnNodesForDifferentChains$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -1062,7 +1062,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_TwoMessagesOneSourceChainCursed$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests @@ -1078,7 +1078,7 @@ runner-test-matrix: - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_GlobalCurseTwoMessagesOnTwoLanes$ path: integration-tests/smoke/ccip/ccip_rmn_test.go test_env_type: docker - runs_on: ubuntu-latest + runs_on: ubuntu20.04-8cores-32GB triggers: - PR E2E Core Tests - Nightly E2E Tests diff --git a/integration-tests/smoke/ccip/ccip_rmn_test.go b/integration-tests/smoke/ccip/ccip_rmn_test.go index c22f9bcf20e..1075b28e9d3 100644 --- a/integration-tests/smoke/ccip/ccip_rmn_test.go +++ b/integration-tests/smoke/ccip/ccip_rmn_test.go @@ -18,11 +18,12 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/chainlink-ccip/pkg/reader" "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/osutil" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" + "github.com/smartcontractkit/chainlink-ccip/pkg/reader" + "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" "github.com/smartcontractkit/chainlink/deployment/environment/devenv" @@ -35,7 +36,6 @@ import ( ) func TestRMN_TwoMessagesOnTwoLanesIncludingBatching(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "messages on two lanes including batching", waitForExec: true, @@ -59,7 +59,6 @@ func TestRMN_TwoMessagesOnTwoLanesIncludingBatching(t *testing.T) { } func TestRMN_MultipleMessagesOnOneLaneNoWaitForExec(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "multiple messages for rmn batching inspection and one rmn node down", waitForExec: false, // do not wait for execution reports @@ -82,7 +81,6 @@ func TestRMN_MultipleMessagesOnOneLaneNoWaitForExec(t *testing.T) { } func TestRMN_NotEnoughObservers(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "one message but not enough observers, should not get a commit report", passIfNoCommitAfter: 15 * time.Second, @@ -105,7 +103,6 @@ func TestRMN_NotEnoughObservers(t *testing.T) { } func TestRMN_DifferentSigners(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "different signers and different observers", homeChainConfig: homeChainConfig{ @@ -130,7 +127,6 @@ func TestRMN_DifferentSigners(t *testing.T) { } func TestRMN_NotEnoughSigners(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "different signers and different observers", passIfNoCommitAfter: 15 * time.Second, @@ -156,7 +152,6 @@ func TestRMN_NotEnoughSigners(t *testing.T) { } func TestRMN_DifferentRmnNodesForDifferentChains(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "different rmn nodes support different chains", waitForExec: false, @@ -183,13 +178,15 @@ func TestRMN_DifferentRmnNodesForDifferentChains(t *testing.T) { } func TestRMN_TwoMessagesOneSourceChainCursed(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ - name: "two messages, one source chain is cursed", + name: "two messages, one source chain is cursed the other chain was cursed but curse is revoked", passIfNoCommitAfter: 15 * time.Second, cursedSubjectsPerChain: map[int][]int{ chain1: {chain0}, }, + revokedCursedSubjectsPerChain: map[int]map[int]time.Duration{ + chain0: {globalCurse: 5 * time.Second}, // chain0 will be globally cursed and curse will be revoked later + }, homeChainConfig: homeChainConfig{ f: map[int]int{chain0: 1, chain1: 1}, }, @@ -210,7 +207,6 @@ func TestRMN_TwoMessagesOneSourceChainCursed(t *testing.T) { } func TestRMN_GlobalCurseTwoMessagesOnTwoLanes(t *testing.T) { - t.Skip("This test is flaky and needs to be fixed") runRmnTestCase(t, rmnTestCase{ name: "global curse messages on two lanes", waitForExec: false, @@ -316,6 +312,7 @@ func runRmnTestCase(t *testing.T, tc rmnTestCase) { t.Logf("Sent all messages, seqNumCommit: %v seqNumExec: %v", seqNumCommit, seqNumExec) tc.callContractsToCurseChains(ctx, t, onChainState, envWithRMN) + tc.callContractsToCurseAndRevokeCurse(ctx, t, onChainState, envWithRMN) tc.enableOracles(ctx, t, envWithRMN, disabledNodes) @@ -428,22 +425,25 @@ type rmnTestCase struct { // If set to a positive value, the test will wait for that duration and will assert that commit report was not delivered. passIfNoCommitAfter time.Duration cursedSubjectsPerChain map[int][]int - waitForExec bool - homeChainConfig homeChainConfig - remoteChainsConfig []remoteChainConfig - rmnNodes []rmnNode - messagesToSend []messageToSend + // revokedCursedSubjectsPerChain is used to revoke this specific curses after a timer expires + revokedCursedSubjectsPerChain map[int]map[int]time.Duration // chainIdx -> subjectIdx -> timer to revoke + waitForExec bool + homeChainConfig homeChainConfig + remoteChainsConfig []remoteChainConfig + rmnNodes []rmnNode + messagesToSend []messageToSend // populated fields after environment setup pf testCasePopulatedFields } type testCasePopulatedFields struct { - chainSelectors []uint64 - rmnHomeNodes []rmn_home.RMNHomeNode - rmnRemoteSigners []rmn_remote.RMNRemoteSigner - rmnHomeSourceChains []rmn_home.RMNHomeSourceChain - cursedSubjectsPerChainSel map[uint64][]uint64 + chainSelectors []uint64 + rmnHomeNodes []rmn_home.RMNHomeNode + rmnRemoteSigners []rmn_remote.RMNRemoteSigner + rmnHomeSourceChains []rmn_home.RMNHomeSourceChain + cursedSubjectsPerChainSel map[uint64][]uint64 + revokedCursedSubjectsPerChainSel map[uint64]map[uint64]time.Duration } func (tc *rmnTestCase) populateFields(t *testing.T, envWithRMN changeset.DeployedEnv, rmnCluster devenv.RMNCluster) { @@ -498,6 +498,22 @@ func (tc *rmnTestCase) populateFields(t *testing.T, envWithRMN changeset.Deploye tc.pf.cursedSubjectsPerChainSel[chainSel] = append(tc.pf.cursedSubjectsPerChainSel[chainSel], subjSel) } } + + // populate revoked cursed subjects with actual chain selectors + tc.pf.revokedCursedSubjectsPerChainSel = make(map[uint64]map[uint64]time.Duration) + for chainIdx, subjects := range tc.revokedCursedSubjectsPerChain { + chainSel := tc.pf.chainSelectors[chainIdx] + for subject, revokeAfter := range subjects { + subjSel := uint64(globalCurse) + if subject != globalCurse { + subjSel = tc.pf.chainSelectors[subject] + } + if _, ok := tc.pf.revokedCursedSubjectsPerChainSel[chainSel]; !ok { + tc.pf.revokedCursedSubjectsPerChainSel[chainSel] = make(map[uint64]time.Duration) + } + tc.pf.revokedCursedSubjectsPerChainSel[chainSel][subjSel] = revokeAfter + } + } } func (tc rmnTestCase) validate() error { @@ -645,6 +661,44 @@ func (tc rmnTestCase) callContractsToCurseChains(ctx context.Context, t *testing } } +func (tc rmnTestCase) callContractsToCurseAndRevokeCurse(ctx context.Context, t *testing.T, onChainState changeset.CCIPOnChainState, envWithRMN changeset.DeployedEnv) { + for _, remoteCfg := range tc.remoteChainsConfig { + remoteSel := tc.pf.chainSelectors[remoteCfg.chainIdx] + chState, ok := onChainState.Chains[remoteSel] + require.True(t, ok) + chain, ok := envWithRMN.Env.Chains[remoteSel] + require.True(t, ok) + + cursedSubjects, ok := tc.revokedCursedSubjectsPerChain[remoteCfg.chainIdx] + if !ok { + continue // nothing to curse on this chain + } + + for subjectDescription, revokeAfter := range cursedSubjects { + subj := reader.GlobalCurseSubject + if subjectDescription != globalCurse { + subj = chainSelectorToBytes16(tc.pf.chainSelectors[subjectDescription]) + } + t.Logf("cursing subject %d (%d)", subj, subjectDescription) + txCurse, errCurse := chState.RMNRemote.Curse(chain.DeployerKey, subj) + _, errConfirm := deployment.ConfirmIfNoError(chain, txCurse, errCurse) + require.NoError(t, errConfirm) + + go func() { + <-time.NewTimer(revokeAfter).C + t.Logf("revoking curse on subject %d (%d)", subj, subjectDescription) + txUncurse, errUncurse := chState.RMNRemote.Uncurse(chain.DeployerKey, subj) + _, errConfirm = deployment.ConfirmIfNoError(chain, txUncurse, errUncurse) + require.NoError(t, errConfirm) + }() + } + + cs, err := chState.RMNRemote.GetCursedSubjects(&bind.CallOpts{Context: ctx}) + require.NoError(t, err) + t.Logf("Cursed subjects: %v", cs) + } +} + func (tc rmnTestCase) enableOracles(ctx context.Context, t *testing.T, envWithRMN changeset.DeployedEnv, nodeIDs []string) { for _, n := range nodeIDs { _, err := envWithRMN.Env.Offchain.EnableNode(ctx, &node.EnableNodeRequest{Id: n})