From a97f0ca524223efc33215f3ef69652d3df5565a2 Mon Sep 17 00:00:00 2001 From: Francesco4203 <100074926+Francesco4203@users.noreply.github.com> Date: Fri, 8 Nov 2024 17:47:07 +0700 Subject: [PATCH] core, triedb/pathdb: pbss fix release v1.13.8 (continue) and v1.14.0 (#626) * core/rawdb: improve state scheme checking (#28724) This pull request improves the condition to check if path state scheme is in use. Originally, root node presence was used as the indicator if path scheme is used or not. However due to fact that root node will be deleted during the initial snap sync, this condition is no longer useful. If PersistentStateID is present, it shows that we've already configured for path scheme. * core, triedb/pathdb: calculate the size for batch pre-allocation (#29106) * core, triedb/pathdb: calculate the size for batch pre-allocation * triedb/pathdb: address comment * triedb/pathdb: fix panic in recoverable (#29107) * triedb/pathdb: fix panic in recoverable * triedb/pathdb: add todo * triedb/pathdb: rename * triedb/pathdb: rename --------- Co-authored-by: rjl493456442 --- core/rawdb/accessors_trie.go | 5 +++++ core/rawdb/schema.go | 26 +++++++++++++------------- core/rawdb/schema_test.go | 4 ++-- trie/triedb/pathdb/database.go | 13 +++++++++---- trie/triedb/pathdb/nodebuffer.go | 15 ++++++++++++++- 5 files changed, 43 insertions(+), 20 deletions(-) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index 14b3a96ba..869991a38 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -294,6 +294,11 @@ func ReadStateScheme(db ethdb.Reader) string { if len(blob) != 0 { return PathScheme } + // The root node might be deleted during the initial snap sync, check + // the persistent state id then. + if id := ReadPersistentStateID(db); id != 0 { + return PathScheme + } // In a hash-based scheme, the genesis state is consistently stored // on the disk. To assess the scheme of the persistent state, it // suffices to inspect the scheme of the genesis state. diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 3c99d1db6..c1ea97e16 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -114,8 +114,8 @@ var ( dirtyAccountsKey = []byte("dacc") // dirtyAccountsPrefix + block hash -> dirty accounts // Path-based storage scheme of merkle patricia trie. - trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node - trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node + TrieNodeAccountPrefix = []byte("A") // TrieNodeAccountPrefix + hexPath -> trie node + TrieNodeStoragePrefix = []byte("O") // TrieNodeStoragePrefix + accountHash + hexPath -> trie node stateIDPrefix = []byte("L") // stateIDPrefix + state root -> state id PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage @@ -250,12 +250,12 @@ func genesisStateSpecKey(hash common.Hash) []byte { // accountTrieNodeKey = trieNodeAccountPrefix + nodePath. func accountTrieNodeKey(path []byte) []byte { - return append(trieNodeAccountPrefix, path...) + return append(TrieNodeAccountPrefix, path...) } -// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath. +// storageTrieNodeKey = TrieNodeStoragePrefix + accountHash + nodePath. func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { - return append(append(trieNodeStoragePrefix, accountHash.Bytes()...), path...) + return append(append(TrieNodeStoragePrefix, accountHash.Bytes()...), path...) } func snapshotConsortiumKey(hash common.Hash) []byte { @@ -277,16 +277,16 @@ func IsLegacyTrieNode(key []byte, val []byte) bool { // account trie node in path-based state scheme, and returns the resolved // node path if so. func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) { - if !bytes.HasPrefix(key, trieNodeAccountPrefix) { + if !bytes.HasPrefix(key, TrieNodeAccountPrefix) { return false, nil } // The remaining key should only consist a hex node path // whose length is in the range 0 to 64 (64 is excluded // since leaves are always wrapped with shortNode). - if len(key) >= len(trieNodeAccountPrefix)+common.HashLength*2 { + if len(key) >= len(TrieNodeAccountPrefix)+common.HashLength*2 { return false, nil } - return true, key[len(trieNodeAccountPrefix):] + return true, key[len(TrieNodeAccountPrefix):] } // IsAccountTrieNode reports whether a provided database entry is an account @@ -300,20 +300,20 @@ func IsAccountTrieNode(key []byte) bool { // trie node in path-based state scheme, and returns the resolved account hash // and node path if so. func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) { - if !bytes.HasPrefix(key, trieNodeStoragePrefix) { + if !bytes.HasPrefix(key, TrieNodeStoragePrefix) { return false, common.Hash{}, nil } // The remaining key consists of 2 parts: // - 32 bytes account hash // - hex node path whose length is in the range 0 to 64 - if len(key) < len(trieNodeStoragePrefix)+common.HashLength { + if len(key) < len(TrieNodeStoragePrefix)+common.HashLength { return false, common.Hash{}, nil } - if len(key) >= len(trieNodeStoragePrefix)+common.HashLength+common.HashLength*2 { + if len(key) >= len(TrieNodeStoragePrefix)+common.HashLength+common.HashLength*2 { return false, common.Hash{}, nil } - accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength]) - return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:] + accountHash := common.BytesToHash(key[len(TrieNodeStoragePrefix) : len(TrieNodeStoragePrefix)+common.HashLength]) + return true, accountHash, key[len(TrieNodeStoragePrefix)+common.HashLength:] } // IsStorageTrieNode reports whether a provided database entry is a storage diff --git a/core/rawdb/schema_test.go b/core/rawdb/schema_test.go index 06b017d41..11d036756 100644 --- a/core/rawdb/schema_test.go +++ b/core/rawdb/schema_test.go @@ -99,7 +99,7 @@ func TestResolveAccountTrieNodeKey(t *testing.T) { }, { name: "storage prefixed", - inputKey: append(trieNodeStoragePrefix, bytes4...), + inputKey: append(TrieNodeStoragePrefix, bytes4...), expectedCheck: false, expectedKey: nil, }, @@ -175,7 +175,7 @@ func TestResolveStorageTrieNode(t *testing.T) { }, { name: "storage prefixed hash 20 length 4", - inputKey: append(append(trieNodeStoragePrefix, bytes20...), bytes4...), + inputKey: append(append(TrieNodeStoragePrefix, bytes20...), bytes4...), expectedCheck: false, expectedHash: common.Hash{}, expectedKey: nil, diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index 873418c9f..ea5c50ab6 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -380,18 +380,23 @@ func (db *Database) Recoverable(root common.Hash) bool { if *id >= dl.stateID() { return false } - + // This is a temporary workaround for the unavailability of the freezer in + // dev mode. As a consequence, the Pathdb loses the ability for deep reorg + // in certain cases. + // TODO(rjl493456442): Implement the in-memory ancient store. + if db.freezer == nil { + return false + } // Ensure the requested state is a canonical state and all state // histories in range [id+1, disklayer.ID] are present and complete. - parent := root return checkHistories(db.freezer, *id+1, dl.stateID()-*id, func(m *meta) error { - if m.parent != parent { + if m.parent != root { return errors.New("unexpected state history") } if len(m.incomplete) > 0 { return errors.New("incomplete state history") } - parent = m.root + root = m.root return nil }) == nil } diff --git a/trie/triedb/pathdb/nodebuffer.go b/trie/triedb/pathdb/nodebuffer.go index b024986d3..0a334baba 100644 --- a/trie/triedb/pathdb/nodebuffer.go +++ b/trie/triedb/pathdb/nodebuffer.go @@ -205,6 +205,19 @@ func (b *nodebuffer) setSize(size int, db ethdb.KeyValueStore, clean *fastcache. return b.flush(db, clean, id, false) } +// allocBatch returns a database batch with pre-allocated buffer. +func (b *nodebuffer) allocBatch(db ethdb.KeyValueStore) ethdb.Batch { + var metasize int + for owner, nodes := range b.nodes { + if owner == (common.Hash{}) { + metasize += len(nodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix + } else { + metasize += len(nodes) * (len(rawdb.TrieNodeStoragePrefix) + common.HashLength) // database key prefix + owner + } + } + return db.NewBatchWithSize((metasize + int(b.size)) * 11 / 10) // extra 10% for potential pebble internal stuff +} + // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error { @@ -218,7 +231,7 @@ func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id ui } var ( start = time.Now() - batch = db.NewBatchWithSize(int(b.size)) + batch = b.allocBatch(db) ) nodes := writeNodes(batch, b.nodes, clean) rawdb.WritePersistentStateID(batch, id)