From d08b3c37f83a85ef7d0d386bae94567d42883f5b Mon Sep 17 00:00:00 2001 From: Valentin Staykov Date: Thu, 28 Nov 2024 15:59:22 +0000 Subject: [PATCH] refactor(stage_interhashes): separate functions into smaller ones and remove repeating code --- .../gointerfaces/sentry/sentry_client_mock.go | 703 ------------------ .../kv/remotedbserver/snapshots_mock.go | 77 -- .../data_stream_server_mock.go | 0 .../stream_server_mock.go | 0 zk/smt/account_processor.go | 161 ++++ zk/smt/account_processor_test.go | 140 ++++ zk/smt/increment_smt.go | 130 ++++ zk/smt/mocks/account_collector_db_mock.go | 116 +++ zk/smt/regenerate_smt.go | 138 ++++ zk/stages/stage_data_stream_catch_up_test.go | 2 +- zk/stages/stage_interhashes.go | 376 +--------- zk/stages/stage_sequence_execute_blocks.go | 3 +- 12 files changed, 690 insertions(+), 1156 deletions(-) delete mode 100644 erigon-lib/gointerfaces/sentry/sentry_client_mock.go delete mode 100644 erigon-lib/kv/remotedbserver/snapshots_mock.go rename zk/datastream/{mock_services => mocks}/data_stream_server_mock.go (100%) rename zk/datastream/{mock_services => mocks}/stream_server_mock.go (100%) create mode 100644 zk/smt/account_processor.go create mode 100644 zk/smt/account_processor_test.go create mode 100644 zk/smt/increment_smt.go create mode 100644 zk/smt/mocks/account_collector_db_mock.go create mode 100644 zk/smt/regenerate_smt.go diff --git a/erigon-lib/gointerfaces/sentry/sentry_client_mock.go b/erigon-lib/gointerfaces/sentry/sentry_client_mock.go deleted file mode 100644 index f17b8731e8a..00000000000 --- a/erigon-lib/gointerfaces/sentry/sentry_client_mock.go +++ /dev/null @@ -1,703 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ledgerwatch/erigon-lib/gointerfaces/sentry (interfaces: SentryClient) -// -// Generated by this command: -// -// mockgen -typed=true -destination=./sentry_client_mock.go -package=sentry . SentryClient -// - -// Package sentry is a generated GoMock package. -package sentry - -import ( - context "context" - reflect "reflect" - - types "github.com/ledgerwatch/erigon-lib/gointerfaces/types" - gomock "go.uber.org/mock/gomock" - grpc "google.golang.org/grpc" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -// MockSentryClient is a mock of SentryClient interface. -type MockSentryClient struct { - ctrl *gomock.Controller - recorder *MockSentryClientMockRecorder -} - -// MockSentryClientMockRecorder is the mock recorder for MockSentryClient. -type MockSentryClientMockRecorder struct { - mock *MockSentryClient -} - -// NewMockSentryClient creates a new mock instance. -func NewMockSentryClient(ctrl *gomock.Controller) *MockSentryClient { - mock := &MockSentryClient{ctrl: ctrl} - mock.recorder = &MockSentryClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSentryClient) EXPECT() *MockSentryClientMockRecorder { - return m.recorder -} - -// AddPeer mocks base method. -func (m *MockSentryClient) AddPeer(arg0 context.Context, arg1 *AddPeerRequest, arg2 ...grpc.CallOption) (*AddPeerReply, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "AddPeer", varargs...) - ret0, _ := ret[0].(*AddPeerReply) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// AddPeer indicates an expected call of AddPeer. -func (mr *MockSentryClientMockRecorder) AddPeer(arg0, arg1 any, arg2 ...any) *MockSentryClientAddPeerCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddPeer", reflect.TypeOf((*MockSentryClient)(nil).AddPeer), varargs...) - return &MockSentryClientAddPeerCall{Call: call} -} - -// MockSentryClientAddPeerCall wrap *gomock.Call -type MockSentryClientAddPeerCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientAddPeerCall) Return(arg0 *AddPeerReply, arg1 error) *MockSentryClientAddPeerCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientAddPeerCall) Do(f func(context.Context, *AddPeerRequest, ...grpc.CallOption) (*AddPeerReply, error)) *MockSentryClientAddPeerCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientAddPeerCall) DoAndReturn(f func(context.Context, *AddPeerRequest, ...grpc.CallOption) (*AddPeerReply, error)) *MockSentryClientAddPeerCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// HandShake mocks base method. -func (m *MockSentryClient) HandShake(arg0 context.Context, arg1 *emptypb.Empty, arg2 ...grpc.CallOption) (*HandShakeReply, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "HandShake", varargs...) - ret0, _ := ret[0].(*HandShakeReply) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// HandShake indicates an expected call of HandShake. -func (mr *MockSentryClientMockRecorder) HandShake(arg0, arg1 any, arg2 ...any) *MockSentryClientHandShakeCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandShake", reflect.TypeOf((*MockSentryClient)(nil).HandShake), varargs...) - return &MockSentryClientHandShakeCall{Call: call} -} - -// MockSentryClientHandShakeCall wrap *gomock.Call -type MockSentryClientHandShakeCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientHandShakeCall) Return(arg0 *HandShakeReply, arg1 error) *MockSentryClientHandShakeCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientHandShakeCall) Do(f func(context.Context, *emptypb.Empty, ...grpc.CallOption) (*HandShakeReply, error)) *MockSentryClientHandShakeCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientHandShakeCall) DoAndReturn(f func(context.Context, *emptypb.Empty, ...grpc.CallOption) (*HandShakeReply, error)) *MockSentryClientHandShakeCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Messages mocks base method. -func (m *MockSentryClient) Messages(arg0 context.Context, arg1 *MessagesRequest, arg2 ...grpc.CallOption) (Sentry_MessagesClient, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Messages", varargs...) - ret0, _ := ret[0].(Sentry_MessagesClient) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Messages indicates an expected call of Messages. -func (mr *MockSentryClientMockRecorder) Messages(arg0, arg1 any, arg2 ...any) *MockSentryClientMessagesCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Messages", reflect.TypeOf((*MockSentryClient)(nil).Messages), varargs...) - return &MockSentryClientMessagesCall{Call: call} -} - -// MockSentryClientMessagesCall wrap *gomock.Call -type MockSentryClientMessagesCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientMessagesCall) Return(arg0 Sentry_MessagesClient, arg1 error) *MockSentryClientMessagesCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientMessagesCall) Do(f func(context.Context, *MessagesRequest, ...grpc.CallOption) (Sentry_MessagesClient, error)) *MockSentryClientMessagesCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientMessagesCall) DoAndReturn(f func(context.Context, *MessagesRequest, ...grpc.CallOption) (Sentry_MessagesClient, error)) *MockSentryClientMessagesCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// NodeInfo mocks base method. -func (m *MockSentryClient) NodeInfo(arg0 context.Context, arg1 *emptypb.Empty, arg2 ...grpc.CallOption) (*types.NodeInfoReply, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "NodeInfo", varargs...) - ret0, _ := ret[0].(*types.NodeInfoReply) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// NodeInfo indicates an expected call of NodeInfo. -func (mr *MockSentryClientMockRecorder) NodeInfo(arg0, arg1 any, arg2 ...any) *MockSentryClientNodeInfoCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeInfo", reflect.TypeOf((*MockSentryClient)(nil).NodeInfo), varargs...) - return &MockSentryClientNodeInfoCall{Call: call} -} - -// MockSentryClientNodeInfoCall wrap *gomock.Call -type MockSentryClientNodeInfoCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientNodeInfoCall) Return(arg0 *types.NodeInfoReply, arg1 error) *MockSentryClientNodeInfoCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientNodeInfoCall) Do(f func(context.Context, *emptypb.Empty, ...grpc.CallOption) (*types.NodeInfoReply, error)) *MockSentryClientNodeInfoCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientNodeInfoCall) DoAndReturn(f func(context.Context, *emptypb.Empty, ...grpc.CallOption) (*types.NodeInfoReply, error)) *MockSentryClientNodeInfoCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// PeerById mocks base method. -func (m *MockSentryClient) PeerById(arg0 context.Context, arg1 *PeerByIdRequest, arg2 ...grpc.CallOption) (*PeerByIdReply, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "PeerById", varargs...) - ret0, _ := ret[0].(*PeerByIdReply) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// PeerById indicates an expected call of PeerById. -func (mr *MockSentryClientMockRecorder) PeerById(arg0, arg1 any, arg2 ...any) *MockSentryClientPeerByIdCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeerById", reflect.TypeOf((*MockSentryClient)(nil).PeerById), varargs...) - return &MockSentryClientPeerByIdCall{Call: call} -} - -// MockSentryClientPeerByIdCall wrap *gomock.Call -type MockSentryClientPeerByIdCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientPeerByIdCall) Return(arg0 *PeerByIdReply, arg1 error) *MockSentryClientPeerByIdCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientPeerByIdCall) Do(f func(context.Context, *PeerByIdRequest, ...grpc.CallOption) (*PeerByIdReply, error)) *MockSentryClientPeerByIdCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientPeerByIdCall) DoAndReturn(f func(context.Context, *PeerByIdRequest, ...grpc.CallOption) (*PeerByIdReply, error)) *MockSentryClientPeerByIdCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// PeerCount mocks base method. -func (m *MockSentryClient) PeerCount(arg0 context.Context, arg1 *PeerCountRequest, arg2 ...grpc.CallOption) (*PeerCountReply, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "PeerCount", varargs...) - ret0, _ := ret[0].(*PeerCountReply) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// PeerCount indicates an expected call of PeerCount. -func (mr *MockSentryClientMockRecorder) PeerCount(arg0, arg1 any, arg2 ...any) *MockSentryClientPeerCountCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeerCount", reflect.TypeOf((*MockSentryClient)(nil).PeerCount), varargs...) - return &MockSentryClientPeerCountCall{Call: call} -} - -// MockSentryClientPeerCountCall wrap *gomock.Call -type MockSentryClientPeerCountCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientPeerCountCall) Return(arg0 *PeerCountReply, arg1 error) *MockSentryClientPeerCountCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientPeerCountCall) Do(f func(context.Context, *PeerCountRequest, ...grpc.CallOption) (*PeerCountReply, error)) *MockSentryClientPeerCountCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientPeerCountCall) DoAndReturn(f func(context.Context, *PeerCountRequest, ...grpc.CallOption) (*PeerCountReply, error)) *MockSentryClientPeerCountCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// PeerEvents mocks base method. -func (m *MockSentryClient) PeerEvents(arg0 context.Context, arg1 *PeerEventsRequest, arg2 ...grpc.CallOption) (Sentry_PeerEventsClient, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "PeerEvents", varargs...) - ret0, _ := ret[0].(Sentry_PeerEventsClient) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// PeerEvents indicates an expected call of PeerEvents. -func (mr *MockSentryClientMockRecorder) PeerEvents(arg0, arg1 any, arg2 ...any) *MockSentryClientPeerEventsCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeerEvents", reflect.TypeOf((*MockSentryClient)(nil).PeerEvents), varargs...) - return &MockSentryClientPeerEventsCall{Call: call} -} - -// MockSentryClientPeerEventsCall wrap *gomock.Call -type MockSentryClientPeerEventsCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientPeerEventsCall) Return(arg0 Sentry_PeerEventsClient, arg1 error) *MockSentryClientPeerEventsCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientPeerEventsCall) Do(f func(context.Context, *PeerEventsRequest, ...grpc.CallOption) (Sentry_PeerEventsClient, error)) *MockSentryClientPeerEventsCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientPeerEventsCall) DoAndReturn(f func(context.Context, *PeerEventsRequest, ...grpc.CallOption) (Sentry_PeerEventsClient, error)) *MockSentryClientPeerEventsCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// PeerMinBlock mocks base method. -func (m *MockSentryClient) PeerMinBlock(arg0 context.Context, arg1 *PeerMinBlockRequest, arg2 ...grpc.CallOption) (*emptypb.Empty, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "PeerMinBlock", varargs...) - ret0, _ := ret[0].(*emptypb.Empty) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// PeerMinBlock indicates an expected call of PeerMinBlock. -func (mr *MockSentryClientMockRecorder) PeerMinBlock(arg0, arg1 any, arg2 ...any) *MockSentryClientPeerMinBlockCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeerMinBlock", reflect.TypeOf((*MockSentryClient)(nil).PeerMinBlock), varargs...) - return &MockSentryClientPeerMinBlockCall{Call: call} -} - -// MockSentryClientPeerMinBlockCall wrap *gomock.Call -type MockSentryClientPeerMinBlockCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientPeerMinBlockCall) Return(arg0 *emptypb.Empty, arg1 error) *MockSentryClientPeerMinBlockCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientPeerMinBlockCall) Do(f func(context.Context, *PeerMinBlockRequest, ...grpc.CallOption) (*emptypb.Empty, error)) *MockSentryClientPeerMinBlockCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientPeerMinBlockCall) DoAndReturn(f func(context.Context, *PeerMinBlockRequest, ...grpc.CallOption) (*emptypb.Empty, error)) *MockSentryClientPeerMinBlockCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Peers mocks base method. -func (m *MockSentryClient) Peers(arg0 context.Context, arg1 *emptypb.Empty, arg2 ...grpc.CallOption) (*PeersReply, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Peers", varargs...) - ret0, _ := ret[0].(*PeersReply) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Peers indicates an expected call of Peers. -func (mr *MockSentryClientMockRecorder) Peers(arg0, arg1 any, arg2 ...any) *MockSentryClientPeersCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Peers", reflect.TypeOf((*MockSentryClient)(nil).Peers), varargs...) - return &MockSentryClientPeersCall{Call: call} -} - -// MockSentryClientPeersCall wrap *gomock.Call -type MockSentryClientPeersCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientPeersCall) Return(arg0 *PeersReply, arg1 error) *MockSentryClientPeersCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientPeersCall) Do(f func(context.Context, *emptypb.Empty, ...grpc.CallOption) (*PeersReply, error)) *MockSentryClientPeersCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientPeersCall) DoAndReturn(f func(context.Context, *emptypb.Empty, ...grpc.CallOption) (*PeersReply, error)) *MockSentryClientPeersCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// PenalizePeer mocks base method. -func (m *MockSentryClient) PenalizePeer(arg0 context.Context, arg1 *PenalizePeerRequest, arg2 ...grpc.CallOption) (*emptypb.Empty, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "PenalizePeer", varargs...) - ret0, _ := ret[0].(*emptypb.Empty) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// PenalizePeer indicates an expected call of PenalizePeer. -func (mr *MockSentryClientMockRecorder) PenalizePeer(arg0, arg1 any, arg2 ...any) *MockSentryClientPenalizePeerCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PenalizePeer", reflect.TypeOf((*MockSentryClient)(nil).PenalizePeer), varargs...) - return &MockSentryClientPenalizePeerCall{Call: call} -} - -// MockSentryClientPenalizePeerCall wrap *gomock.Call -type MockSentryClientPenalizePeerCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientPenalizePeerCall) Return(arg0 *emptypb.Empty, arg1 error) *MockSentryClientPenalizePeerCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientPenalizePeerCall) Do(f func(context.Context, *PenalizePeerRequest, ...grpc.CallOption) (*emptypb.Empty, error)) *MockSentryClientPenalizePeerCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientPenalizePeerCall) DoAndReturn(f func(context.Context, *PenalizePeerRequest, ...grpc.CallOption) (*emptypb.Empty, error)) *MockSentryClientPenalizePeerCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// SendMessageById mocks base method. -func (m *MockSentryClient) SendMessageById(arg0 context.Context, arg1 *SendMessageByIdRequest, arg2 ...grpc.CallOption) (*SentPeers, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "SendMessageById", varargs...) - ret0, _ := ret[0].(*SentPeers) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SendMessageById indicates an expected call of SendMessageById. -func (mr *MockSentryClientMockRecorder) SendMessageById(arg0, arg1 any, arg2 ...any) *MockSentryClientSendMessageByIdCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessageById", reflect.TypeOf((*MockSentryClient)(nil).SendMessageById), varargs...) - return &MockSentryClientSendMessageByIdCall{Call: call} -} - -// MockSentryClientSendMessageByIdCall wrap *gomock.Call -type MockSentryClientSendMessageByIdCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientSendMessageByIdCall) Return(arg0 *SentPeers, arg1 error) *MockSentryClientSendMessageByIdCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientSendMessageByIdCall) Do(f func(context.Context, *SendMessageByIdRequest, ...grpc.CallOption) (*SentPeers, error)) *MockSentryClientSendMessageByIdCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientSendMessageByIdCall) DoAndReturn(f func(context.Context, *SendMessageByIdRequest, ...grpc.CallOption) (*SentPeers, error)) *MockSentryClientSendMessageByIdCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// SendMessageByMinBlock mocks base method. -func (m *MockSentryClient) SendMessageByMinBlock(arg0 context.Context, arg1 *SendMessageByMinBlockRequest, arg2 ...grpc.CallOption) (*SentPeers, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "SendMessageByMinBlock", varargs...) - ret0, _ := ret[0].(*SentPeers) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SendMessageByMinBlock indicates an expected call of SendMessageByMinBlock. -func (mr *MockSentryClientMockRecorder) SendMessageByMinBlock(arg0, arg1 any, arg2 ...any) *MockSentryClientSendMessageByMinBlockCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessageByMinBlock", reflect.TypeOf((*MockSentryClient)(nil).SendMessageByMinBlock), varargs...) - return &MockSentryClientSendMessageByMinBlockCall{Call: call} -} - -// MockSentryClientSendMessageByMinBlockCall wrap *gomock.Call -type MockSentryClientSendMessageByMinBlockCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientSendMessageByMinBlockCall) Return(arg0 *SentPeers, arg1 error) *MockSentryClientSendMessageByMinBlockCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientSendMessageByMinBlockCall) Do(f func(context.Context, *SendMessageByMinBlockRequest, ...grpc.CallOption) (*SentPeers, error)) *MockSentryClientSendMessageByMinBlockCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientSendMessageByMinBlockCall) DoAndReturn(f func(context.Context, *SendMessageByMinBlockRequest, ...grpc.CallOption) (*SentPeers, error)) *MockSentryClientSendMessageByMinBlockCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// SendMessageToAll mocks base method. -func (m *MockSentryClient) SendMessageToAll(arg0 context.Context, arg1 *OutboundMessageData, arg2 ...grpc.CallOption) (*SentPeers, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "SendMessageToAll", varargs...) - ret0, _ := ret[0].(*SentPeers) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SendMessageToAll indicates an expected call of SendMessageToAll. -func (mr *MockSentryClientMockRecorder) SendMessageToAll(arg0, arg1 any, arg2 ...any) *MockSentryClientSendMessageToAllCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessageToAll", reflect.TypeOf((*MockSentryClient)(nil).SendMessageToAll), varargs...) - return &MockSentryClientSendMessageToAllCall{Call: call} -} - -// MockSentryClientSendMessageToAllCall wrap *gomock.Call -type MockSentryClientSendMessageToAllCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientSendMessageToAllCall) Return(arg0 *SentPeers, arg1 error) *MockSentryClientSendMessageToAllCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientSendMessageToAllCall) Do(f func(context.Context, *OutboundMessageData, ...grpc.CallOption) (*SentPeers, error)) *MockSentryClientSendMessageToAllCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientSendMessageToAllCall) DoAndReturn(f func(context.Context, *OutboundMessageData, ...grpc.CallOption) (*SentPeers, error)) *MockSentryClientSendMessageToAllCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// SendMessageToRandomPeers mocks base method. -func (m *MockSentryClient) SendMessageToRandomPeers(arg0 context.Context, arg1 *SendMessageToRandomPeersRequest, arg2 ...grpc.CallOption) (*SentPeers, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "SendMessageToRandomPeers", varargs...) - ret0, _ := ret[0].(*SentPeers) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SendMessageToRandomPeers indicates an expected call of SendMessageToRandomPeers. -func (mr *MockSentryClientMockRecorder) SendMessageToRandomPeers(arg0, arg1 any, arg2 ...any) *MockSentryClientSendMessageToRandomPeersCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessageToRandomPeers", reflect.TypeOf((*MockSentryClient)(nil).SendMessageToRandomPeers), varargs...) - return &MockSentryClientSendMessageToRandomPeersCall{Call: call} -} - -// MockSentryClientSendMessageToRandomPeersCall wrap *gomock.Call -type MockSentryClientSendMessageToRandomPeersCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientSendMessageToRandomPeersCall) Return(arg0 *SentPeers, arg1 error) *MockSentryClientSendMessageToRandomPeersCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientSendMessageToRandomPeersCall) Do(f func(context.Context, *SendMessageToRandomPeersRequest, ...grpc.CallOption) (*SentPeers, error)) *MockSentryClientSendMessageToRandomPeersCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientSendMessageToRandomPeersCall) DoAndReturn(f func(context.Context, *SendMessageToRandomPeersRequest, ...grpc.CallOption) (*SentPeers, error)) *MockSentryClientSendMessageToRandomPeersCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// SetStatus mocks base method. -func (m *MockSentryClient) SetStatus(arg0 context.Context, arg1 *StatusData, arg2 ...grpc.CallOption) (*SetStatusReply, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "SetStatus", varargs...) - ret0, _ := ret[0].(*SetStatusReply) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SetStatus indicates an expected call of SetStatus. -func (mr *MockSentryClientMockRecorder) SetStatus(arg0, arg1 any, arg2 ...any) *MockSentryClientSetStatusCall { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetStatus", reflect.TypeOf((*MockSentryClient)(nil).SetStatus), varargs...) - return &MockSentryClientSetStatusCall{Call: call} -} - -// MockSentryClientSetStatusCall wrap *gomock.Call -type MockSentryClientSetStatusCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSentryClientSetStatusCall) Return(arg0 *SetStatusReply, arg1 error) *MockSentryClientSetStatusCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSentryClientSetStatusCall) Do(f func(context.Context, *StatusData, ...grpc.CallOption) (*SetStatusReply, error)) *MockSentryClientSetStatusCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSentryClientSetStatusCall) DoAndReturn(f func(context.Context, *StatusData, ...grpc.CallOption) (*SetStatusReply, error)) *MockSentryClientSetStatusCall { - c.Call = c.Call.DoAndReturn(f) - return c -} diff --git a/erigon-lib/kv/remotedbserver/snapshots_mock.go b/erigon-lib/kv/remotedbserver/snapshots_mock.go deleted file mode 100644 index cdd66394fec..00000000000 --- a/erigon-lib/kv/remotedbserver/snapshots_mock.go +++ /dev/null @@ -1,77 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ledgerwatch/erigon-lib/kv/remotedbserver (interfaces: Snapshots) -// -// Generated by this command: -// -// mockgen -typed=true -destination=./snapshots_mock.go -package=remotedbserver . Snapshots -// - -// Package remotedbserver is a generated GoMock package. -package remotedbserver - -import ( - reflect "reflect" - - gomock "go.uber.org/mock/gomock" -) - -// MockSnapshots is a mock of Snapshots interface. -type MockSnapshots struct { - ctrl *gomock.Controller - recorder *MockSnapshotsMockRecorder -} - -// MockSnapshotsMockRecorder is the mock recorder for MockSnapshots. -type MockSnapshotsMockRecorder struct { - mock *MockSnapshots -} - -// NewMockSnapshots creates a new mock instance. -func NewMockSnapshots(ctrl *gomock.Controller) *MockSnapshots { - mock := &MockSnapshots{ctrl: ctrl} - mock.recorder = &MockSnapshotsMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSnapshots) EXPECT() *MockSnapshotsMockRecorder { - return m.recorder -} - -// Files mocks base method. -func (m *MockSnapshots) Files() []string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Files") - ret0, _ := ret[0].([]string) - return ret0 -} - -// Files indicates an expected call of Files. -func (mr *MockSnapshotsMockRecorder) Files() *MockSnapshotsFilesCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Files", reflect.TypeOf((*MockSnapshots)(nil).Files)) - return &MockSnapshotsFilesCall{Call: call} -} - -// MockSnapshotsFilesCall wrap *gomock.Call -type MockSnapshotsFilesCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSnapshotsFilesCall) Return(arg0 []string) *MockSnapshotsFilesCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSnapshotsFilesCall) Do(f func() []string) *MockSnapshotsFilesCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSnapshotsFilesCall) DoAndReturn(f func() []string) *MockSnapshotsFilesCall { - c.Call = c.Call.DoAndReturn(f) - return c -} diff --git a/zk/datastream/mock_services/data_stream_server_mock.go b/zk/datastream/mocks/data_stream_server_mock.go similarity index 100% rename from zk/datastream/mock_services/data_stream_server_mock.go rename to zk/datastream/mocks/data_stream_server_mock.go diff --git a/zk/datastream/mock_services/stream_server_mock.go b/zk/datastream/mocks/stream_server_mock.go similarity index 100% rename from zk/datastream/mock_services/stream_server_mock.go rename to zk/datastream/mocks/stream_server_mock.go diff --git a/zk/smt/account_processor.go b/zk/smt/account_processor.go new file mode 100644 index 00000000000..eb565ae378e --- /dev/null +++ b/zk/smt/account_processor.go @@ -0,0 +1,161 @@ +package smt + +import ( + "fmt" + "math/big" + "strings" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types/accounts" + "github.com/ledgerwatch/erigon/smt/pkg/utils" + "github.com/status-im/keycard-go/hexutils" +) + +//go:generate mockgen -typed=true -destination=./mocks/account_collector_db_mock.go -package=mocks . AccountCollectorDb + +type AccountCollectorDb interface { + InsertKeySource(key utils.NodeKey, value []byte) error + InsertAccountValue(key utils.NodeKey, value utils.NodeValue8) error +} + +type accountCollector struct { + db AccountCollectorDb + psr *state.PlainStateReader + keys []utils.NodeKey +} + +func NewAccountCollector(db AccountCollectorDb, psr *state.PlainStateReader) *accountCollector { + return &accountCollector{ + db: db, + psr: psr, + keys: []utils.NodeKey{}, + } +} + +// parses the data and saves it to the DB +// collects the saved keys in the array "keys" +func (ac *accountCollector) processAccount(a *accounts.Account, as map[string]string, inc uint64, addr common.Address) error { + // get the account balance and nonce + if err := ac.insertAccountStateToKV(addr.String(), a.Balance.ToBig(), new(big.Int).SetUint64(a.Nonce)); err != nil { + return fmt.Errorf("insertAccountStateToKV: %w", err) + } + + // store the contract bytecode + cc, err := ac.psr.ReadAccountCode(addr, inc, a.CodeHash) + if err != nil { + return fmt.Errorf("ReadAccountCode: %w", err) + } + + if len(cc) > 0 { + ach := hexutils.BytesToHex(cc) + hexcc := "0x" + ach + if err = ac.insertContractBytecodeToKV(addr.String(), hexcc); err != nil { + return fmt.Errorf("insertContractBytecodeToKV: %w", err) + } + } + + if len(as) > 0 { + // store the account storage + if err = ac.insertContractStorageToKV(addr.String(), as); err != nil { + return fmt.Errorf("insertContractStorageToKV: %w", err) + } + } + + return nil +} + +func (ac *accountCollector) insertContractBytecodeToKV(ethAddr string, bytecode string) error { + keyContractCode := utils.KeyContractCode(ethAddr) + keyContractLength := utils.KeyContractLength(ethAddr) + bi := utils.HashContractBytecodeBigInt(bytecode) + + parsedBytecode := strings.TrimPrefix(bytecode, "0x") + if len(parsedBytecode)%2 != 0 { + parsedBytecode = "0" + parsedBytecode + } + + bytecodeLength := len(parsedBytecode) / 2 + + if err := ac.parseAndInsertKV(utils.SC_CODE, ethAddr, keyContractCode, bi, common.Hash{}); err != nil { + return fmt.Errorf("parseAndInsertKV: %w", err) + } + + if err := ac.parseAndInsertKV(utils.SC_LENGTH, ethAddr, keyContractLength, big.NewInt(int64(bytecodeLength)), common.Hash{}); err != nil { + return fmt.Errorf("parseAndInsertKV: %w", err) + } + + return nil +} + +func (ac *accountCollector) insertContractStorageToKV(ethAddr string, storage map[string]string) (err error) { + add := utils.ScalarToArrayBig(utils.ConvertHexToBigInt(ethAddr)) + + var keyStoragePosition utils.NodeKey + for k, v := range storage { + if v == "" { + continue + } + + keyStoragePosition = utils.KeyContractStorage(add, k) + + base := 10 + if strings.HasPrefix(v, "0x") { + v = v[2:] + base = 16 + } + + val, _ := new(big.Int).SetString(v, base) + sp, _ := utils.StrValToBigInt(k) + if err := ac.parseAndInsertKV(utils.SC_STORAGE, ethAddr, keyStoragePosition, val, common.BigToHash(sp)); err != nil { + return fmt.Errorf("parseAndInsertKV: %w", err) + } + } + + return nil +} + +func (ac *accountCollector) insertAccountStateToKV(ethAddr string, balance, nonce *big.Int) error { + keyBalance := utils.KeyEthAddrBalance(ethAddr) + keyNonce := utils.KeyEthAddrNonce(ethAddr) + + if err := ac.parseAndInsertKV(utils.KEY_BALANCE, ethAddr, keyBalance, balance, common.Hash{}); err != nil { + return fmt.Errorf("parseAndInsertKV: %w", err) + } + + if err := ac.parseAndInsertKV(utils.KEY_NONCE, ethAddr, keyNonce, nonce, common.Hash{}); err != nil { + return fmt.Errorf("parseAndInsertKV: %w", err) + } + return nil +} + +func (ac *accountCollector) parseAndInsertKV(encodeKey int, ethAddr string, key utils.NodeKey, val *big.Int, storagePos common.Hash) error { + x := utils.ScalarToArrayBig(val) + valueNode, err := utils.NodeValue8FromBigIntArray(x) + if err != nil { + return fmt.Errorf("NodeValue8FromBigIntArray: %w", err) + } + + if !valueNode.IsZero() { + if err := ac.insertKVtoDb(encodeKey, ethAddr, key, valueNode, storagePos); err != nil { + return fmt.Errorf("processAccount: %w", err) + } + } + + return nil +} + +func (ac *accountCollector) insertKVtoDb(encodeKey int, ethAddr string, key utils.NodeKey, val *utils.NodeValue8, storagePos common.Hash) error { + if err := ac.db.InsertAccountValue(key, *val); err != nil { + return fmt.Errorf("InsertAccountValue: %w", err) + } + + ks := utils.EncodeKeySource(encodeKey, utils.ConvertHexToAddress(ethAddr), storagePos) + if err := ac.db.InsertKeySource(key, ks); err != nil { + return fmt.Errorf("InsertKeySource: %w", err) + } + + ac.keys = append(ac.keys, key) + + return nil +} diff --git a/zk/smt/account_processor_test.go b/zk/smt/account_processor_test.go new file mode 100644 index 00000000000..ab06e59d23e --- /dev/null +++ b/zk/smt/account_processor_test.go @@ -0,0 +1,140 @@ +package smt + +import ( + "context" + "errors" + "testing" + + "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/smt/pkg/utils" + "github.com/ledgerwatch/erigon/zk/smt/mocks" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" +) + +type insertKeySourceParams struct { + key utils.NodeKey + value []byte + result error +} + +type insertAccountValueParams struct { + key utils.NodeKey + value utils.NodeValue8 + result error +} +type testStruct struct { + name string + encodeKey int + ethAddress string + key utils.NodeKey + val utils.NodeValue8 + insertKeySourceParams *insertKeySourceParams + insertAccountValueParams *insertAccountValueParams + resultError string +} + +func Test_insertKVtoDb(t *testing.T) { + tests := []testStruct{ + { + name: "inserted key in keys array", + encodeKey: 1, + ethAddress: "0x1234567890abcdef", + key: utils.NodeKey{1, 2, 3, 4}, + val: utils.NodeValue8{}, + insertKeySourceParams: nil, + insertAccountValueParams: nil, + resultError: "", + }, + { + name: "InsertAccountValue inserts correct values", + encodeKey: 1, + ethAddress: "0x1234567890abcdef", + key: utils.NodeKey{1, 2, 3, 4}, + val: utils.NodeValue8{}, + insertAccountValueParams: &insertAccountValueParams{ + key: utils.NodeKey{1, 2, 3, 4}, + value: utils.NodeValue8{}, + result: nil, + }, + insertKeySourceParams: nil, + resultError: "", + }, + { + name: "InsertAccountValue returns err", + encodeKey: 1, + ethAddress: "0x1234567890abcdef", + key: utils.NodeKey{1, 2, 3, 4}, + val: utils.NodeValue8{}, + insertAccountValueParams: &insertAccountValueParams{ + key: utils.NodeKey{1, 2, 3, 4}, + value: utils.NodeValue8{}, + result: errors.New("error"), + }, + insertKeySourceParams: nil, + resultError: "InsertAccountValue", + }, + { + name: "InsertKeySource inserts correct values", + encodeKey: 1, + ethAddress: "0x1234567890abcdef", + key: utils.NodeKey{1, 2, 3, 4}, + val: utils.NodeValue8{}, + insertAccountValueParams: nil, + insertKeySourceParams: &insertKeySourceParams{ + key: utils.NodeKey{1, 2, 3, 4}, + value: []byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 52, 86, 120, 144, 171, 205, 239}, + result: nil, + }, + resultError: "", + }, + { + name: "InsertKeySource returns err", + encodeKey: 1, + ethAddress: "0x1234567890abcdef", + key: utils.NodeKey{1, 2, 3, 4}, + val: utils.NodeValue8{}, + insertAccountValueParams: nil, + insertKeySourceParams: &insertKeySourceParams{ + key: utils.NodeKey{1, 2, 3, 4}, + value: []byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 52, 86, 120, 144, 171, 205, 239}, + result: errors.New("error"), + }, + resultError: "InsertKeySource", + }, + } + db := memdb.NewTestDB(t) + defer db.Close() + tx, err := db.BeginRw(context.Background()) + assert.NoError(t, err) + defer tx.Rollback() + ctrl := gomock.NewController(t) + psr := state.NewPlainStateReader(tx) + defer ctrl.Finish() + + for _, tc := range tests { + eriDb := mocks.NewMockAccountCollectorDb(ctrl) + if tc.insertKeySourceParams != nil { + eriDb.EXPECT().InsertKeySource(tc.insertKeySourceParams.key, tc.insertKeySourceParams.value).Return(tc.insertKeySourceParams.result).Times(1) + } else { + eriDb.EXPECT().InsertKeySource(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + } + if tc.insertAccountValueParams != nil { + eriDb.EXPECT().InsertAccountValue(tc.insertAccountValueParams.key, tc.insertAccountValueParams.value).Return(tc.insertAccountValueParams.result).Times(1) + } else { + eriDb.EXPECT().InsertAccountValue(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + } + + accCollector := NewAccountCollector(eriDb, psr) + err = accCollector.insertKVtoDb(tc.encodeKey, tc.ethAddress, tc.key, &tc.val) + if tc.resultError != "" { + assert.ErrorContains(t, err, tc.resultError) + assert.Equal(t, 0, len(accCollector.keys)) + } else { + assert.NoError(t, err) + assert.Equal(t, 1, len(accCollector.keys)) + assert.Equal(t, tc.key, accCollector.keys[0]) + } + } +} diff --git a/zk/smt/increment_smt.go b/zk/smt/increment_smt.go new file mode 100644 index 00000000000..379e4a9ce08 --- /dev/null +++ b/zk/smt/increment_smt.go @@ -0,0 +1,130 @@ +package smt + +import ( + "context" + "fmt" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/dbutils" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/systemcontracts" + "github.com/ledgerwatch/erigon/core/types/accounts" + "github.com/ledgerwatch/erigon/eth/stagedsync" + eridb "github.com/ledgerwatch/erigon/smt/pkg/db" + "github.com/ledgerwatch/erigon/smt/pkg/smt" + "github.com/ledgerwatch/erigon/turbo/trie" + "github.com/ledgerwatch/erigon/zk/hermez_db" + "github.com/ledgerwatch/erigon/zkevm/log" + "github.com/status-im/keycard-go/hexutils" +) + +func IncrementIntermediateHashes(ctx context.Context, logPrefix string, s *stagedsync.StageState, db kv.RwTx, eridb *eridb.EriDb, dbSmt *smt.SMT, from, to uint64) (common.Hash, error) { + log.Info(fmt.Sprintf("[%s] Increment trie hashes started", logPrefix), "previousRootHeight", s.BlockNumber, "calculatingRootHeight", to) + defer log.Info(fmt.Sprintf("[%s] Increment ended", logPrefix)) + + ac, err := db.CursorDupSort(kv.AccountChangeSet) + if err != nil { + return trie.EmptyRoot, err + } + defer ac.Close() + + sc, err := db.CursorDupSort(kv.StorageChangeSet) + if err != nil { + return trie.EmptyRoot, err + } + defer sc.Close() + + // progress printer + accChanges := make(map[common.Address]*accounts.Account) + codeChanges := make(map[common.Address]string) + storageChanges := make(map[common.Address]map[string]string) + + // case when we are incrementing from block 1 + // we chould include the 0 block which is the genesis data + if from != 0 { + from += 1 + } + + // NB: changeset tables are zero indexed + // changeset tables contain historical value at N-1, so we look up values from plainstate + // i+1 to get state at the beginning of the next batch + psr := state.NewPlainState(db, from+1, systemcontracts.SystemContractCodeLookup["Hermez"]) + defer psr.Close() + + for i := from; i <= to; i++ { + dupSortKey := dbutils.EncodeBlockNumber(i) + psr.SetBlockNr(i + 1) + + // collect changes to accounts and code + for _, v, err := ac.SeekExact(dupSortKey); err == nil && v != nil; _, v, err = ac.NextDup() { + addr := common.BytesToAddress(v[:length.Addr]) + + currAcc, err := psr.ReadAccountData(addr) + if err != nil { + return trie.EmptyRoot, err + } + + // store the account + accChanges[addr] = currAcc + + cc, err := psr.ReadAccountCode(addr, currAcc.Incarnation, currAcc.CodeHash) + if err != nil { + return trie.EmptyRoot, err + } + + ach := hexutils.BytesToHex(cc) + if len(ach) > 0 { + hexcc := "0x" + ach + codeChanges[addr] = hexcc + } + } + + err = db.ForPrefix(kv.StorageChangeSet, dupSortKey, func(sk, sv []byte) error { + changesetKey := sk[length.BlockNum:] + address, incarnation := dbutils.PlainParseStoragePrefix(changesetKey) + + sstorageKey := sv[:length.Hash] + stk := common.BytesToHash(sstorageKey) + + value, err := psr.ReadAccountStorage(address, incarnation, &stk) + if err != nil { + return err + } + + stkk := fmt.Sprintf("0x%032x", stk) + v := fmt.Sprintf("0x%032x", common.BytesToHash(value)) + + m := make(map[string]string) + m[stkk] = v + + if storageChanges[address] == nil { + storageChanges[address] = make(map[string]string) + } + storageChanges[address][stkk] = v + return nil + }) + if err != nil { + return trie.EmptyRoot, err + } + } + + if _, _, err := dbSmt.SetStorage(ctx, logPrefix, accChanges, codeChanges, storageChanges); err != nil { + return trie.EmptyRoot, err + } + + log.Info(fmt.Sprintf("[%s] Regeneration trie hashes finished. Commiting batch", logPrefix)) + + lr := dbSmt.LastRoot() + + hash := common.BigToHash(lr) + + // do not put this outside, because sequencer uses this function to calculate root for each block + hermezDb := hermez_db.NewHermezDb(db) + if err := hermezDb.WriteSmtDepth(to, uint64(dbSmt.GetDepth())); err != nil { + return trie.EmptyRoot, err + } + + return hash, nil +} diff --git a/zk/smt/mocks/account_collector_db_mock.go b/zk/smt/mocks/account_collector_db_mock.go new file mode 100644 index 00000000000..35078bef9ba --- /dev/null +++ b/zk/smt/mocks/account_collector_db_mock.go @@ -0,0 +1,116 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/ledgerwatch/erigon/zk/smt (interfaces: AccountCollectorDb) +// +// Generated by this command: +// +// mockgen -typed=true -destination=./mocks/account_collector_db_mock.go -package=mocks . AccountCollectorDb +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + utils "github.com/ledgerwatch/erigon/smt/pkg/utils" + gomock "go.uber.org/mock/gomock" +) + +// MockAccountCollectorDb is a mock of AccountCollectorDb interface. +type MockAccountCollectorDb struct { + ctrl *gomock.Controller + recorder *MockAccountCollectorDbMockRecorder +} + +// MockAccountCollectorDbMockRecorder is the mock recorder for MockAccountCollectorDb. +type MockAccountCollectorDbMockRecorder struct { + mock *MockAccountCollectorDb +} + +// NewMockAccountCollectorDb creates a new mock instance. +func NewMockAccountCollectorDb(ctrl *gomock.Controller) *MockAccountCollectorDb { + mock := &MockAccountCollectorDb{ctrl: ctrl} + mock.recorder = &MockAccountCollectorDbMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAccountCollectorDb) EXPECT() *MockAccountCollectorDbMockRecorder { + return m.recorder +} + +// InsertAccountValue mocks base method. +func (m *MockAccountCollectorDb) InsertAccountValue(arg0 utils.NodeKey, arg1 utils.NodeValue8) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InsertAccountValue", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// InsertAccountValue indicates an expected call of InsertAccountValue. +func (mr *MockAccountCollectorDbMockRecorder) InsertAccountValue(arg0, arg1 any) *MockAccountCollectorDbInsertAccountValueCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertAccountValue", reflect.TypeOf((*MockAccountCollectorDb)(nil).InsertAccountValue), arg0, arg1) + return &MockAccountCollectorDbInsertAccountValueCall{Call: call} +} + +// MockAccountCollectorDbInsertAccountValueCall wrap *gomock.Call +type MockAccountCollectorDbInsertAccountValueCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockAccountCollectorDbInsertAccountValueCall) Return(arg0 error) *MockAccountCollectorDbInsertAccountValueCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockAccountCollectorDbInsertAccountValueCall) Do(f func(utils.NodeKey, utils.NodeValue8) error) *MockAccountCollectorDbInsertAccountValueCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockAccountCollectorDbInsertAccountValueCall) DoAndReturn(f func(utils.NodeKey, utils.NodeValue8) error) *MockAccountCollectorDbInsertAccountValueCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// InsertKeySource mocks base method. +func (m *MockAccountCollectorDb) InsertKeySource(arg0 utils.NodeKey, arg1 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InsertKeySource", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// InsertKeySource indicates an expected call of InsertKeySource. +func (mr *MockAccountCollectorDbMockRecorder) InsertKeySource(arg0, arg1 any) *MockAccountCollectorDbInsertKeySourceCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertKeySource", reflect.TypeOf((*MockAccountCollectorDb)(nil).InsertKeySource), arg0, arg1) + return &MockAccountCollectorDbInsertKeySourceCall{Call: call} +} + +// MockAccountCollectorDbInsertKeySourceCall wrap *gomock.Call +type MockAccountCollectorDbInsertKeySourceCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockAccountCollectorDbInsertKeySourceCall) Return(arg0 error) *MockAccountCollectorDbInsertKeySourceCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockAccountCollectorDbInsertKeySourceCall) Do(f func(utils.NodeKey, []byte) error) *MockAccountCollectorDbInsertKeySourceCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockAccountCollectorDbInsertKeySourceCall) DoAndReturn(f func(utils.NodeKey, []byte) error) *MockAccountCollectorDbInsertKeySourceCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/zk/smt/regenerate_smt.go b/zk/smt/regenerate_smt.go new file mode 100644 index 00000000000..7ee0903b9ec --- /dev/null +++ b/zk/smt/regenerate_smt.go @@ -0,0 +1,138 @@ +package smt + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/ledgerwatch/erigon-lib/kv/dbutils" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types/accounts" + "github.com/ledgerwatch/erigon/eth/stagedsync/stages" + db2 "github.com/ledgerwatch/erigon/smt/pkg/db" + "github.com/ledgerwatch/erigon/smt/pkg/smt" + "github.com/ledgerwatch/erigon/turbo/trie" + "github.com/ledgerwatch/erigon/zk" + "github.com/ledgerwatch/erigon/zk/hermez_db" + "github.com/ledgerwatch/erigon/zkevm/log" +) + +func RegenerateIntermediateHashes(ctx context.Context, logPrefix string, db kv.RwTx, eridb *db2.EriDb, smtIn *smt.SMT, toBlock uint64) (common.Hash, error) { + log.Info(fmt.Sprintf("[%s] Regeneration trie hashes started", logPrefix)) + defer log.Info(fmt.Sprintf("[%s] Regeneration ended", logPrefix)) + + if err := stages.SaveStageProgress(db, stages.IntermediateHashes, 0); err != nil { + log.Warn(fmt.Sprint("regenerate SaveStageProgress to zero error: ", err)) + } + + var a *accounts.Account + var addr common.Address + var as map[string]string + var inc uint64 + + psr := state.NewPlainStateReader(db) + + accCollector := NewAccountCollector(eridb, psr) + + log.Info(fmt.Sprintf("[%s] Collecting account data...", logPrefix)) + dataCollectStartTime := time.Now() + + // get total accounts count for progress printer + total := uint64(0) + if err := psr.ForEach(kv.PlainState, nil, func(k, acc []byte) error { + total++ + return nil + }); err != nil { + return trie.EmptyRoot, err + } + + progressChan, stopProgressPrinter := zk.ProgressPrinterWithoutValues(fmt.Sprintf("[%s] SMT regenerate progress", logPrefix), total*2) + defer stopProgressPrinter() + + progCt := uint64(0) + err := psr.ForEach(kv.PlainState, nil, func(k, acc []byte) error { + progCt++ + progressChan <- progCt + var err error + if len(k) == 20 { + if a != nil { // don't run process on first loop for first account (or it will miss collecting storage) + if err = accCollector.processAccount(a, as, inc, addr); err != nil { + return err + } + } + + a = &accounts.Account{} + + if err := a.DecodeForStorage(acc); err != nil { + // TODO: not an account? + as = make(map[string]string) + return nil + } + addr = common.BytesToAddress(k) + inc = a.Incarnation + // empty storage of previous account + as = make(map[string]string) + } else { // otherwise we're reading storage + _, incarnation, key := dbutils.PlainParseCompositeStorageKey(k) + if incarnation != inc { + return nil + } + + sk := fmt.Sprintf("0x%032x", key) + v := fmt.Sprintf("0x%032x", acc) + + as[sk] = trimHexString(v) + } + return nil + }) + + stopProgressPrinter() + + if err != nil { + return trie.EmptyRoot, err + } + + // process the final account + if err := accCollector.processAccount(a, as, inc, addr); err != nil { + return trie.EmptyRoot, err + } + + dataCollectTime := time.Since(dataCollectStartTime) + log.Info(fmt.Sprintf("[%s] Collecting account data finished in %v", logPrefix, dataCollectTime)) + + // generate tree + if _, err := smtIn.GenerateFromKVBulk(ctx, logPrefix, accCollector.keys); err != nil { + return trie.EmptyRoot, err + } + + err2 := db.ClearBucket("HermezSmtAccountValues") + if err2 != nil { + log.Warn(fmt.Sprint("regenerate SaveStageProgress to zero error: ", err2)) + } + + root := smtIn.LastRoot() + + // save it here so we don't + hermezDb := hermez_db.NewHermezDb(db) + if err := hermezDb.WriteSmtDepth(toBlock, uint64(smtIn.GetDepth())); err != nil { + return trie.EmptyRoot, err + } + + return common.BigToHash(root), nil +} + +func trimHexString(s string) string { + s = strings.TrimPrefix(s, "0x") + + for i := 0; i < len(s); i++ { + if s[i] != '0' { + return "0x" + s[i:] + } + } + + return "0x0" +} diff --git a/zk/stages/stage_data_stream_catch_up_test.go b/zk/stages/stage_data_stream_catch_up_test.go index 00b1fb880d6..a2b19506e3d 100644 --- a/zk/stages/stage_data_stream_catch_up_test.go +++ b/zk/stages/stage_data_stream_catch_up_test.go @@ -14,7 +14,7 @@ import ( "github.com/ledgerwatch/erigon/eth/stagedsync" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/smt/pkg/db" - mocks "github.com/ledgerwatch/erigon/zk/datastream/mock_services" + mocks "github.com/ledgerwatch/erigon/zk/datastream/mocks" "github.com/ledgerwatch/erigon/zk/hermez_db" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" diff --git a/zk/stages/stage_interhashes.go b/zk/stages/stage_interhashes.go index c2f708f6967..aac395b8db1 100644 --- a/zk/stages/stage_interhashes.go +++ b/zk/stages/stage_interhashes.go @@ -4,38 +4,25 @@ import ( "fmt" "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/state" - state2 "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/types" db2 "github.com/ledgerwatch/erigon/smt/pkg/db" "github.com/ledgerwatch/erigon/smt/pkg/smt" - "github.com/ledgerwatch/erigon/smt/pkg/utils" "github.com/ledgerwatch/erigon/zk/hermez_db" "github.com/ledgerwatch/log/v3" - "strings" - "context" - "math/big" - "time" "os" - "github.com/ledgerwatch/erigon-lib/kv/dbutils" "github.com/ledgerwatch/erigon/core/rawdb" - "github.com/ledgerwatch/erigon/core/systemcontracts" - "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/stagedsync" - "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/stages/headerdownload" "github.com/ledgerwatch/erigon/turbo/trie" - "github.com/ledgerwatch/erigon/zk" zkSmt "github.com/ledgerwatch/erigon/zk/smt" - "github.com/status-im/keycard-go/hexutils" ) type ZkInterHashesCfg struct { @@ -139,11 +126,11 @@ func SpawnZkIntermediateHashesStage(s *stagedsync.StageState, u stagedsync.Unwin if shouldIncrementBecauseOfAFlag { log.Debug(fmt.Sprintf("[%s] IncrementTreeAlways true - incrementing tree", logPrefix), "previousRootHeight", s.BlockNumber, "calculatingRootHeight", to) } - if root, err = zkIncrementIntermediateHashes(ctx, logPrefix, s, tx, eridb, smt, s.BlockNumber, to); err != nil { + if root, err = zkSmt.IncrementIntermediateHashes(ctx, logPrefix, s, tx, eridb, smt, s.BlockNumber, to); err != nil { return trie.EmptyRoot, err } } else { - if root, err = regenerateIntermediateHashes(ctx, logPrefix, tx, eridb, smt, to); err != nil { + if root, err = zkSmt.RegenerateIntermediateHashes(ctx, logPrefix, tx, eridb, smt, to); err != nil { return trie.EmptyRoot, err } } @@ -232,362 +219,3 @@ func UnwindZkIntermediateHashesStage(u *stagedsync.UnwindState, s *stagedsync.St } return nil } - -func regenerateIntermediateHashes(ctx context.Context, logPrefix string, db kv.RwTx, eridb *db2.EriDb, smtIn *smt.SMT, toBlock uint64) (common.Hash, error) { - log.Info(fmt.Sprintf("[%s] Regeneration trie hashes started", logPrefix)) - defer log.Info(fmt.Sprintf("[%s] Regeneration ended", logPrefix)) - - if err := stages.SaveStageProgress(db, stages.IntermediateHashes, 0); err != nil { - log.Warn(fmt.Sprint("regenerate SaveStageProgress to zero error: ", err)) - } - - var a *accounts.Account - var addr common.Address - var as map[string]string - var inc uint64 - - psr := state2.NewPlainStateReader(db) - - log.Info(fmt.Sprintf("[%s] Collecting account data...", logPrefix)) - dataCollectStartTime := time.Now() - keys := []utils.NodeKey{} - - // get total accounts count for progress printer - total := uint64(0) - if err := psr.ForEach(kv.PlainState, nil, func(k, acc []byte) error { - total++ - return nil - }); err != nil { - return trie.EmptyRoot, err - } - - progressChan, stopProgressPrinter := zk.ProgressPrinterWithoutValues(fmt.Sprintf("[%s] SMT regenerate progress", logPrefix), total*2) - - progCt := uint64(0) - err := psr.ForEach(kv.PlainState, nil, func(k, acc []byte) error { - progCt++ - progressChan <- progCt - var err error - if len(k) == 20 { - if a != nil { // don't run process on first loop for first account (or it will miss collecting storage) - keys, err = processAccount(eridb, a, as, inc, psr, addr, keys) - if err != nil { - return err - } - } - - a = &accounts.Account{} - - if err := a.DecodeForStorage(acc); err != nil { - // TODO: not an account? - as = make(map[string]string) - return nil - } - addr = common.BytesToAddress(k) - inc = a.Incarnation - // empty storage of previous account - as = make(map[string]string) - } else { // otherwise we're reading storage - _, incarnation, key := dbutils.PlainParseCompositeStorageKey(k) - if incarnation != inc { - return nil - } - - sk := fmt.Sprintf("0x%032x", key) - v := fmt.Sprintf("0x%032x", acc) - - as[sk] = TrimHexString(v) - } - return nil - }) - - stopProgressPrinter() - - if err != nil { - return trie.EmptyRoot, err - } - - // process the final account - keys, err = processAccount(eridb, a, as, inc, psr, addr, keys) - if err != nil { - return trie.EmptyRoot, err - } - - dataCollectTime := time.Since(dataCollectStartTime) - log.Info(fmt.Sprintf("[%s] Collecting account data finished in %v", logPrefix, dataCollectTime)) - - // generate tree - if _, err := smtIn.GenerateFromKVBulk(ctx, logPrefix, keys); err != nil { - return trie.EmptyRoot, err - } - - err2 := db.ClearBucket("HermezSmtAccountValues") - if err2 != nil { - log.Warn(fmt.Sprint("regenerate SaveStageProgress to zero error: ", err2)) - } - - root := smtIn.LastRoot() - - // save it here so we don't - hermezDb := hermez_db.NewHermezDb(db) - if err := hermezDb.WriteSmtDepth(toBlock, uint64(smtIn.GetDepth())); err != nil { - return trie.EmptyRoot, err - } - - return common.BigToHash(root), nil -} - -func zkIncrementIntermediateHashes(ctx context.Context, logPrefix string, s *stagedsync.StageState, db kv.RwTx, eridb *db2.EriDb, dbSmt *smt.SMT, from, to uint64) (common.Hash, error) { - log.Info(fmt.Sprintf("[%s] Increment trie hashes started", logPrefix), "previousRootHeight", s.BlockNumber, "calculatingRootHeight", to) - defer log.Info(fmt.Sprintf("[%s] Increment ended", logPrefix)) - - ac, err := db.CursorDupSort(kv.AccountChangeSet) - if err != nil { - return trie.EmptyRoot, err - } - defer ac.Close() - - sc, err := db.CursorDupSort(kv.StorageChangeSet) - if err != nil { - return trie.EmptyRoot, err - } - defer sc.Close() - - // progress printer - accChanges := make(map[common.Address]*accounts.Account) - codeChanges := make(map[common.Address]string) - storageChanges := make(map[common.Address]map[string]string) - - // case when we are incrementing from block 1 - // we chould include the 0 block which is the genesis data - if from != 0 { - from += 1 - } - - // NB: changeset tables are zero indexed - // changeset tables contain historical value at N-1, so we look up values from plainstate - // i+1 to get state at the beginning of the next batch - psr := state2.NewPlainState(db, from+1, systemcontracts.SystemContractCodeLookup["Hermez"]) - defer psr.Close() - - for i := from; i <= to; i++ { - dupSortKey := dbutils.EncodeBlockNumber(i) - psr.SetBlockNr(i + 1) - - // collect changes to accounts and code - for _, v, err := ac.SeekExact(dupSortKey); err == nil && v != nil; _, v, err = ac.NextDup() { - addr := common.BytesToAddress(v[:length.Addr]) - - currAcc, err := psr.ReadAccountData(addr) - if err != nil { - return trie.EmptyRoot, err - } - - // store the account - accChanges[addr] = currAcc - - cc, err := psr.ReadAccountCode(addr, currAcc.Incarnation, currAcc.CodeHash) - if err != nil { - return trie.EmptyRoot, err - } - - ach := hexutils.BytesToHex(cc) - if len(ach) > 0 { - hexcc := "0x" + ach - codeChanges[addr] = hexcc - } - } - - err = db.ForPrefix(kv.StorageChangeSet, dupSortKey, func(sk, sv []byte) error { - changesetKey := sk[length.BlockNum:] - address, incarnation := dbutils.PlainParseStoragePrefix(changesetKey) - - sstorageKey := sv[:length.Hash] - stk := common.BytesToHash(sstorageKey) - - value, err := psr.ReadAccountStorage(address, incarnation, &stk) - if err != nil { - return err - } - - stkk := fmt.Sprintf("0x%032x", stk) - v := fmt.Sprintf("0x%032x", common.BytesToHash(value)) - - m := make(map[string]string) - m[stkk] = v - - if storageChanges[address] == nil { - storageChanges[address] = make(map[string]string) - } - storageChanges[address][stkk] = v - return nil - }) - if err != nil { - return trie.EmptyRoot, err - } - } - - if _, _, err := dbSmt.SetStorage(ctx, logPrefix, accChanges, codeChanges, storageChanges); err != nil { - return trie.EmptyRoot, err - } - - log.Info(fmt.Sprintf("[%s] Regeneration trie hashes finished. Commiting batch", logPrefix)) - - lr := dbSmt.LastRoot() - - hash := common.BigToHash(lr) - - // do not put this outside, because sequencer uses this function to calculate root for each block - hermezDb := hermez_db.NewHermezDb(db) - if err := hermezDb.WriteSmtDepth(to, uint64(dbSmt.GetDepth())); err != nil { - return trie.EmptyRoot, err - } - - return hash, nil -} - -func processAccount(db smt.DB, a *accounts.Account, as map[string]string, inc uint64, psr *state2.PlainStateReader, addr common.Address, keys []utils.NodeKey) ([]utils.NodeKey, error) { - // get the account balance and nonce - keys, err := insertAccountStateToKV(db, keys, addr.String(), a.Balance.ToBig(), new(big.Int).SetUint64(a.Nonce)) - if err != nil { - return []utils.NodeKey{}, err - } - - // store the contract bytecode - cc, err := psr.ReadAccountCode(addr, inc, a.CodeHash) - if err != nil { - return []utils.NodeKey{}, err - } - - ach := hexutils.BytesToHex(cc) - if len(ach) > 0 { - hexcc := "0x" + ach - keys, err = insertContractBytecodeToKV(db, keys, addr.String(), hexcc) - if err != nil { - return []utils.NodeKey{}, err - } - } - - if len(as) > 0 { - // store the account storage - keys, err = insertContractStorageToKV(db, keys, addr.String(), as) - if err != nil { - return []utils.NodeKey{}, err - } - } - - return keys, nil -} - -func insertContractBytecodeToKV(db smt.DB, keys []utils.NodeKey, ethAddr string, bytecode string) ([]utils.NodeKey, error) { - keyContractCode := utils.KeyContractCode(ethAddr) - keyContractLength := utils.KeyContractLength(ethAddr) - bi := utils.HashContractBytecodeBigInt(bytecode) - - parsedBytecode := strings.TrimPrefix(bytecode, "0x") - if len(parsedBytecode)%2 != 0 { - parsedBytecode = "0" + parsedBytecode - } - - bytecodeLength := len(parsedBytecode) / 2 - - x := utils.ScalarToArrayBig(bi) - valueContractCode, err := utils.NodeValue8FromBigIntArray(x) - if err != nil { - return []utils.NodeKey{}, err - } - - x = utils.ScalarToArrayBig(big.NewInt(int64(bytecodeLength))) - valueContractLength, err := utils.NodeValue8FromBigIntArray(x) - if err != nil { - return []utils.NodeKey{}, err - } - if !valueContractCode.IsZero() { - keys = append(keys, keyContractCode) - db.InsertAccountValue(keyContractCode, *valueContractCode) - - ks := utils.EncodeKeySource(utils.SC_CODE, utils.ConvertHexToAddress(ethAddr), common.Hash{}) - db.InsertKeySource(keyContractCode, ks) - } - - if !valueContractLength.IsZero() { - keys = append(keys, keyContractLength) - db.InsertAccountValue(keyContractLength, *valueContractLength) - - ks := utils.EncodeKeySource(utils.SC_LENGTH, utils.ConvertHexToAddress(ethAddr), common.Hash{}) - db.InsertKeySource(keyContractLength, ks) - } - - return keys, nil -} - -func insertContractStorageToKV(db smt.DB, keys []utils.NodeKey, ethAddr string, storage map[string]string) ([]utils.NodeKey, error) { - a := utils.ConvertHexToBigInt(ethAddr) - add := utils.ScalarToArrayBig(a) - - for k, v := range storage { - if v == "" { - continue - } - - keyStoragePosition := utils.KeyContractStorage(add, k) - - base := 10 - if strings.HasPrefix(v, "0x") { - v = v[2:] - base = 16 - } - - val, _ := new(big.Int).SetString(v, base) - - x := utils.ScalarToArrayBig(val) - parsedValue, err := utils.NodeValue8FromBigIntArray(x) - if err != nil { - return []utils.NodeKey{}, err - } - if !parsedValue.IsZero() { - keys = append(keys, keyStoragePosition) - db.InsertAccountValue(keyStoragePosition, *parsedValue) - - sp, _ := utils.StrValToBigInt(k) - - ks := utils.EncodeKeySource(utils.SC_STORAGE, utils.ConvertHexToAddress(ethAddr), common.BigToHash(sp)) - db.InsertKeySource(keyStoragePosition, ks) - } - } - - return keys, nil -} - -func insertAccountStateToKV(db smt.DB, keys []utils.NodeKey, ethAddr string, balance, nonce *big.Int) ([]utils.NodeKey, error) { - keyBalance := utils.KeyEthAddrBalance(ethAddr) - keyNonce := utils.KeyEthAddrNonce(ethAddr) - - x := utils.ScalarToArrayBig(balance) - valueBalance, err := utils.NodeValue8FromBigIntArray(x) - if err != nil { - return []utils.NodeKey{}, err - } - - x = utils.ScalarToArrayBig(nonce) - valueNonce, err := utils.NodeValue8FromBigIntArray(x) - if err != nil { - return []utils.NodeKey{}, err - } - - if !valueBalance.IsZero() { - keys = append(keys, keyBalance) - db.InsertAccountValue(keyBalance, *valueBalance) - - ks := utils.EncodeKeySource(utils.KEY_BALANCE, utils.ConvertHexToAddress(ethAddr), common.Hash{}) - db.InsertKeySource(keyBalance, ks) - } - if !valueNonce.IsZero() { - keys = append(keys, keyNonce) - db.InsertAccountValue(keyNonce, *valueNonce) - - ks := utils.EncodeKeySource(utils.KEY_NONCE, utils.ConvertHexToAddress(ethAddr), common.Hash{}) - db.InsertKeySource(keyNonce, ks) - } - return keys, nil -} diff --git a/zk/stages/stage_sequence_execute_blocks.go b/zk/stages/stage_sequence_execute_blocks.go index d801fecf7fe..5fe070a8113 100644 --- a/zk/stages/stage_sequence_execute_blocks.go +++ b/zk/stages/stage_sequence_execute_blocks.go @@ -17,6 +17,7 @@ import ( "github.com/ledgerwatch/erigon/smt/pkg/blockinfo" "github.com/ledgerwatch/erigon/zk/erigon_db" "github.com/ledgerwatch/erigon/zk/hermez_db" + zkSmt "github.com/ledgerwatch/erigon/zk/smt" zktypes "github.com/ledgerwatch/erigon/zk/types" "github.com/ledgerwatch/secp256k1" ) @@ -182,7 +183,7 @@ func finaliseBlock( } // this is actually the interhashes stage - newRoot, err := zkIncrementIntermediateHashes(batchContext.ctx, batchContext.s.LogPrefix(), batchContext.s, batchContext.sdb.tx, batchContext.sdb.eridb, batchContext.sdb.smt, newHeader.Number.Uint64()-1, newHeader.Number.Uint64()) + newRoot, err := zkSmt.IncrementIntermediateHashes(batchContext.ctx, batchContext.s.LogPrefix(), batchContext.s, batchContext.sdb.tx, batchContext.sdb.eridb, batchContext.sdb.smt, newHeader.Number.Uint64()-1, newHeader.Number.Uint64()) if err != nil { return nil, err }