From 662f42351b9346193c5fe0e637e56abefd0f2558 Mon Sep 17 00:00:00 2001 From: rehs0y Date: Mon, 28 Oct 2024 16:15:51 +0200 Subject: [PATCH] optimization: (eventsyncer) don't remove QBFT instances to speed up sync [main] (#1824) * optimization: (eventsyncer) don't remove QBFT instances to speed up historical syncing (#1615) * optimize badger delete action * try another optimization * use dropprefix instead of delete all keys * drop messages only when own share or fullnode * removed usage of store.CleanAllInstances * remove storageMap from eventhandler --------- Co-authored-by: y0sher Co-authored-by: Matus Kysel * fix build and lint --------- Co-authored-by: Anton Korpusenko Co-authored-by: Matus Kysel Co-authored-by: Nikita Kryuchkov --- cli/operator/node.go | 3 --- eth/ethtest/utils_test.go | 2 -- eth/eventhandler/event_handler.go | 4 ---- eth/eventhandler/event_handler_test.go | 2 -- eth/eventhandler/handlers.go | 16 ++++------------ eth/eventsyncer/event_syncer_test.go | 4 ---- go.mod | 2 +- go.sum | 2 ++ ibft/storage/store.go | 2 +- storage/basedb/storage.go | 1 - storage/kv/badger.go | 22 ++++------------------ storage/kv/badger_test.go | 6 ------ 12 files changed, 12 insertions(+), 54 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 8c02d03dab..956e28a85b 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -321,7 +321,6 @@ var StartNodeCmd = &cobra.Command{ logger, executionClient, validatorCtrl, - storageMap, metricsReporter, networkConfig, nodeStorage, @@ -613,7 +612,6 @@ func setupEventHandling( logger *zap.Logger, executionClient *executionclient.ExecutionClient, validatorCtrl validator.Controller, - storageMap *ibftstorage.QBFTStores, metricsReporter metricsreporter.MetricsReporter, networkConfig networkconfig.NetworkConfig, nodeStorage operatorstorage.Storage, @@ -636,7 +634,6 @@ func setupEventHandling( operatorDecrypter, cfg.SSVOptions.ValidatorOptions.KeyManager, cfg.SSVOptions.ValidatorOptions.Beacon, - storageMap, eventhandler.WithFullNode(), eventhandler.WithLogger(logger), eventhandler.WithMetrics(metricsReporter), diff --git a/eth/ethtest/utils_test.go b/eth/ethtest/utils_test.go index ef99d995b4..f05a10ed1a 100644 --- a/eth/ethtest/utils_test.go +++ b/eth/ethtest/utils_test.go @@ -190,7 +190,6 @@ func setupEventHandler( operator.privateKey, keyManager, bc, - storageMap, eventhandler.WithFullNode(), eventhandler.WithLogger(logger), ) @@ -222,7 +221,6 @@ func setupEventHandler( operator.privateKey, keyManager, bc, - storageMap, eventhandler.WithFullNode(), eventhandler.WithLogger(logger), ) diff --git a/eth/eventhandler/event_handler.go b/eth/eventhandler/event_handler.go index 4c64b13268..afacd35f43 100644 --- a/eth/eventhandler/event_handler.go +++ b/eth/eventhandler/event_handler.go @@ -18,7 +18,6 @@ import ( "github.com/bloxapp/ssv/eth/eventparser" "github.com/bloxapp/ssv/eth/executionclient" "github.com/bloxapp/ssv/eth/localevents" - qbftstorage "github.com/bloxapp/ssv/ibft/storage" "github.com/bloxapp/ssv/logging/fields" "github.com/bloxapp/ssv/networkconfig" operatordatastore "github.com/bloxapp/ssv/operator/datastore" @@ -65,7 +64,6 @@ type EventHandler struct { operatorDecrypter keys.OperatorDecrypter keyManager spectypes.KeyManager beacon beaconprotocol.BeaconNode - storageMap *qbftstorage.QBFTStores fullNode bool logger *zap.Logger @@ -81,7 +79,6 @@ func New( operatorDecrypter keys.OperatorDecrypter, keyManager spectypes.KeyManager, beacon beaconprotocol.BeaconNode, - storageMap *qbftstorage.QBFTStores, opts ...Option, ) (*EventHandler, error) { eh := &EventHandler{ @@ -93,7 +90,6 @@ func New( operatorDecrypter: operatorDecrypter, keyManager: keyManager, beacon: beacon, - storageMap: storageMap, logger: zap.NewNop(), metrics: nopMetrics{}, } diff --git a/eth/eventhandler/event_handler_test.go b/eth/eventhandler/event_handler_test.go index 5ab3071778..82e4e8836f 100644 --- a/eth/eventhandler/event_handler_test.go +++ b/eth/eventhandler/event_handler_test.go @@ -1316,7 +1316,6 @@ func setupEventHandler(t *testing.T, ctx context.Context, logger *zap.Logger, ne operator.privateKey, keyManager, bc, - storageMap, WithFullNode(), WithLogger(logger)) if err != nil { @@ -1353,7 +1352,6 @@ func setupEventHandler(t *testing.T, ctx context.Context, logger *zap.Logger, ne operator.privateKey, keyManager, bc, - storageMap, WithFullNode(), WithLogger(logger)) if err != nil { diff --git a/eth/eventhandler/handlers.go b/eth/eventhandler/handlers.go index e92bb4ed6b..913f565d26 100644 --- a/eth/eventhandler/handlers.go +++ b/eth/eventhandler/handlers.go @@ -16,7 +16,6 @@ import ( "github.com/bloxapp/ssv/eth/contract" "github.com/bloxapp/ssv/logging/fields" "github.com/bloxapp/ssv/operator/duties" - qbftstorage "github.com/bloxapp/ssv/protocol/v2/qbft/storage" ssvtypes "github.com/bloxapp/ssv/protocol/v2/types" registrystorage "github.com/bloxapp/ssv/registry/storage" "github.com/bloxapp/ssv/storage/basedb" @@ -224,6 +223,7 @@ func (eh *EventHandler) handleShareCreation( encryptedKeys [][]byte, ) (*ssvtypes.SSVShare, error) { share, shareSecret, err := eh.validatorAddedEventToShare( + txn, validatorEvent, sharePublicKeys, encryptedKeys, @@ -252,6 +252,7 @@ func (eh *EventHandler) handleShareCreation( } func (eh *EventHandler) validatorAddedEventToShare( + txn basedb.Txn, event *contract.ContractValidatorAdded, sharePublicKeys [][]byte, encryptedKeys [][]byte, @@ -341,16 +342,7 @@ func (eh *EventHandler) handleValidatorRemoved(txn basedb.Txn, event *contract.C return nil, &MalformedEventError{Err: ErrShareBelongsToDifferentOwner} } - removeDecidedMessages := func(role spectypes.BeaconRole, store qbftstorage.QBFTStore) error { - messageID := spectypes.NewMsgID(eh.networkConfig.Domain, share.ValidatorPubKey, role) - return store.CleanAllInstances(logger, messageID[:]) - } - err := eh.storageMap.Each(removeDecidedMessages) - if err != nil { - return nil, fmt.Errorf("could not clean all decided messages: %w", err) - } - - if err := eh.nodeStorage.Shares().Delete(txn, share.ValidatorPubKey); err != nil { + if err := eh.nodeStorage.Shares().Delete(txn, share.ValidatorPubKey[:]); err != nil { return nil, fmt.Errorf("could not remove validator share: %w", err) } @@ -359,7 +351,7 @@ func (eh *EventHandler) handleValidatorRemoved(txn basedb.Txn, event *contract.C logger = logger.With(zap.String("validator_pubkey", hex.EncodeToString(share.ValidatorPubKey))) } if isOperatorShare { - err = eh.keyManager.RemoveShare(hex.EncodeToString(share.SharePubKey)) + err := eh.keyManager.RemoveShare(hex.EncodeToString(share.SharePubKey)) if err != nil { return nil, fmt.Errorf("could not remove share from ekm storage: %w", err) } diff --git a/eth/eventsyncer/event_syncer_test.go b/eth/eventsyncer/event_syncer_test.go index 8745f71f8b..a62d9ab692 100644 --- a/eth/eventsyncer/event_syncer_test.go +++ b/eth/eventsyncer/event_syncer_test.go @@ -33,7 +33,6 @@ import ( "github.com/bloxapp/ssv/eth/eventparser" "github.com/bloxapp/ssv/eth/executionclient" "github.com/bloxapp/ssv/eth/simulator/simcontract" - ibftstorage "github.com/bloxapp/ssv/ibft/storage" "github.com/bloxapp/ssv/networkconfig" operatorstorage "github.com/bloxapp/ssv/operator/storage" "github.com/bloxapp/ssv/operator/validator" @@ -137,8 +136,6 @@ func setupEventHandler(t *testing.T, ctx context.Context, logger *zap.Logger) *e }) require.NoError(t, err) - storageMap := ibftstorage.NewStores() - privateKey, err := keys.GeneratePrivateKey() if err != nil { logger.Fatal("failed generating operator key %v", zap.Error(err)) @@ -179,7 +176,6 @@ func setupEventHandler(t *testing.T, ctx context.Context, logger *zap.Logger) *e privateKey, keyManager, bc, - storageMap, eventhandler.WithFullNode(), eventhandler.WithLogger(logger)) diff --git a/go.mod b/go.mod index ee0b1010cd..16f88ea466 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.3.2 github.com/cespare/xxhash/v2 v2.2.0 github.com/cornelk/hashmap v1.0.8 - github.com/dgraph-io/badger/v4 v4.1.0 + github.com/dgraph-io/badger/v4 v4.2.0 github.com/dgraph-io/ristretto v0.1.1 github.com/ethereum/go-ethereum v1.13.5 github.com/ferranbt/fastssz v0.1.3 diff --git a/go.sum b/go.sum index 4676d11d94..ad2c669dc9 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger/v4 v4.1.0 h1:E38jc0f+RATYrycSUf9LMv/t47XAy+3CApyYSq4APOQ= github.com/dgraph-io/badger/v4 v4.1.0/go.mod h1:P50u28d39ibBRmIJuQC/NSdBOg46HnHw7al2SW5QRHg= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= github.com/dgraph-io/ristretto v0.1.1-0.20211108053508-297c39e6640f h1:NBGp2JpfMtXmanFWt6f3gEdBtnLO5LupRvm3w4TXrvs= github.com/dgraph-io/ristretto v0.1.1-0.20211108053508-297c39e6640f/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= diff --git a/ibft/storage/store.go b/ibft/storage/store.go index 32c614a1e6..6836f722e3 100644 --- a/ibft/storage/store.go +++ b/ibft/storage/store.go @@ -139,7 +139,7 @@ func (i *ibftStorage) CleanAllInstances(logger *zap.Logger, msgID []byte) error prefix := i.prefix prefix = append(prefix, msgID[:]...) prefix = append(prefix, []byte(instanceKey)...) - _, err := i.db.DeletePrefix(prefix) + err := i.db.DropPrefix(prefix) if err != nil { return errors.Wrap(err, "failed to remove decided") } diff --git a/storage/basedb/storage.go b/storage/basedb/storage.go index 1f021d84a7..66daa80356 100644 --- a/storage/basedb/storage.go +++ b/storage/basedb/storage.go @@ -53,7 +53,6 @@ type Database interface { // TODO: consider moving these functions into Reader and ReadWriter interfaces? CountPrefix(prefix []byte) (int64, error) - DeletePrefix(prefix []byte) (int, error) DropPrefix(prefix []byte) error Update(fn func(Txn) error) error Close() error diff --git a/storage/kv/badger.go b/storage/kv/badger.go index 0b1dd0492a..7ffc950e27 100644 --- a/storage/kv/badger.go +++ b/storage/kv/badger.go @@ -158,22 +158,6 @@ func (b *BadgerDB) Delete(prefix []byte, key []byte) error { }) } -// DeletePrefix all items with this prefix -func (b *BadgerDB) DeletePrefix(prefix []byte) (int, error) { - count := 0 - err := b.db.Update(func(txn *badger.Txn) error { - rawKeys := b.listRawKeys(prefix, txn) - for _, k := range rawKeys { - if err := txn.Delete(k); err != nil { - return err - } - count++ - } - return nil - }) - return count, err -} - // GetAll returns all the items of a given collection func (b *BadgerDB) GetAll(prefix []byte, handler func(int, basedb.Obj) error) error { // we got issues when reading more than 100 items with iterator (items get mixed up) @@ -249,11 +233,13 @@ func (b *BadgerDB) listRawKeys(prefix []byte, txn *badger.Txn) [][]byte { opt := badger.DefaultIteratorOptions opt.Prefix = prefix + opt.PrefetchValues = false + it := txn.NewIterator(opt) defer it.Close() + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { - item := it.Item() - keys = append(keys, item.KeyCopy(nil)) + keys = append(keys, it.Item().KeyCopy(nil)) } return keys diff --git a/storage/kv/badger_test.go b/storage/kv/badger_test.go index 3ab394602b..8fa3a11099 100644 --- a/storage/kv/badger_test.go +++ b/storage/kv/badger_test.go @@ -87,9 +87,6 @@ func TestBadgerEndToEnd(t *testing.T) { require.False(t, found) require.NoError(t, db.DropPrefix([]byte("prefix2"))) - deleted, err := db.DeletePrefix([]byte("prefix1")) - require.NoError(t, err) - require.Equal(t, 1, deleted) } func TestBadgerDb_GetAll(t *testing.T) { @@ -194,7 +191,4 @@ func getAllTest(t *testing.T, n int, db basedb.Database) { visited[string(item.Key)] = item.Value } require.Equal(t, n, len(visited)) - count, err := db.DeletePrefix(prefix) - require.NoError(t, err) - require.Equal(t, n, count) }