From 5d984796afd4aa7c00c6663f4155488a9df73d0e Mon Sep 17 00:00:00 2001 From: cui <523516579@qq.com> Date: Mon, 19 Feb 2024 20:03:58 +0800 Subject: [PATCH 01/91] core: using math.MaxUint64 instead of 0xffffffffffffffff (#29022) --- core/vm/instructions.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 023aa0af0008..b8055de6bce7 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -17,6 +17,8 @@ package vm import ( + "math" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -359,7 +361,7 @@ func opCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ ) uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() if overflow { - uint64CodeOffset = 0xffffffffffffffff + uint64CodeOffset = math.MaxUint64 } codeCopy := getData(scope.Contract.Code, uint64CodeOffset, length.Uint64()) scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) @@ -377,7 +379,7 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ) uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() if overflow { - uint64CodeOffset = 0xffffffffffffffff + uint64CodeOffset = math.MaxUint64 } addr := common.Address(a.Bytes20()) codeCopy := getData(interpreter.evm.StateDB.GetCode(addr), uint64CodeOffset, length.Uint64()) From 6fb0d0992bd4eb91faf1e081b3c4aa46adb0ef7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 19 Feb 2024 15:59:40 +0200 Subject: [PATCH 02/91] core/txpool, miner: speed up blob pool pending retrievals (#29008) * core/txpool, miner: speed up blob pool pending retrievals * miner: fix test merge issue * eth: same same * core/txpool/blobpool: speed up blobtx creation in benchmark a bit * core/txpool/blobpool: fix linter --------- Co-authored-by: Martin Holst Swende --- core/txpool/blobpool/blobpool.go | 17 ++++---- core/txpool/blobpool/blobpool_test.go | 58 +++++++++++++++++++++++++++ core/txpool/legacypool/legacypool.go | 4 +- core/txpool/subpool.go | 6 +-- eth/handler_test.go | 4 +- miner/ordering.go | 26 +++++++----- miner/ordering_test.go | 9 +++-- miner/worker.go | 18 ++++----- 8 files changed, 105 insertions(+), 37 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index ed561f8186da..0ab382001a10 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1456,13 +1456,14 @@ func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *u pendwaitHist.Update(time.Since(pendStart).Nanoseconds()) defer p.lock.RUnlock() - defer func(start time.Time) { - pendtimeHist.Update(time.Since(start).Nanoseconds()) - }(time.Now()) + execStart := time.Now() + defer func() { + pendtimeHist.Update(time.Since(execStart).Nanoseconds()) + }() - pending := make(map[common.Address][]*txpool.LazyTransaction) + pending := make(map[common.Address][]*txpool.LazyTransaction, len(p.index)) for addr, txs := range p.index { - var lazies []*txpool.LazyTransaction + lazies := make([]*txpool.LazyTransaction, 0, len(txs)) for _, tx := range txs { // If transaction filtering was requested, discard badly priced ones if minTip != nil && baseFee != nil { @@ -1486,9 +1487,9 @@ func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *u lazies = append(lazies, &txpool.LazyTransaction{ Pool: p, Hash: tx.hash, - Time: time.Now(), // TODO(karalabe): Maybe save these and use that? - GasFeeCap: tx.execFeeCap.ToBig(), - GasTipCap: tx.execTipCap.ToBig(), + Time: execStart, // TODO(karalabe): Maybe save these and use that? + GasFeeCap: tx.execFeeCap, + GasTipCap: tx.execTipCap, Gas: tx.execGas, BlobGas: tx.blobGas, }) diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 58353e48289b..4cec78b57263 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -1288,3 +1288,61 @@ func TestAdd(t *testing.T) { pool.Close() } } + +// Benchmarks the time it takes to assemble the lazy pending transaction list +// from the pool contents. +func BenchmarkPoolPending100Mb(b *testing.B) { benchmarkPoolPending(b, 100_000_000) } +func BenchmarkPoolPending1GB(b *testing.B) { benchmarkPoolPending(b, 1_000_000_000) } +func BenchmarkPoolPending10GB(b *testing.B) { benchmarkPoolPending(b, 10_000_000_000) } + +func benchmarkPoolPending(b *testing.B, datacap uint64) { + // Calculate the maximum number of transaction that would fit into the pool + // and generate a set of random accounts to seed them with. + capacity := datacap / params.BlobTxBlobGasPerBlob + + var ( + basefee = uint64(1050) + blobfee = uint64(105) + signer = types.LatestSigner(testChainConfig) + statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) + chain = &testBlockChain{ + config: testChainConfig, + basefee: uint256.NewInt(basefee), + blobfee: uint256.NewInt(blobfee), + statedb: statedb, + } + pool = New(Config{Datadir: ""}, chain) + ) + + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + b.Fatalf("failed to create blob pool: %v", err) + } + // Fill the pool up with one random transaction from each account with the + // same price and everything to maximize the worst case scenario + for i := 0; i < int(capacity); i++ { + blobtx := makeUnsignedTx(0, 10, basefee+10, blobfee) + blobtx.R = uint256.NewInt(1) + blobtx.S = uint256.NewInt(uint64(100 + i)) + blobtx.V = uint256.NewInt(0) + tx := types.NewTx(blobtx) + addr, err := types.Sender(signer, tx) + if err != nil { + b.Fatal(err) + } + statedb.AddBalance(addr, uint256.NewInt(1_000_000_000)) + pool.add(tx) + } + statedb.Commit(0, true) + defer pool.Close() + + // Benchmark assembling the pending + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + p := pool.Pending(uint256.NewInt(1), chain.basefee, chain.blobfee) + if len(p) != int(capacity) { + b.Fatalf("have %d want %d", len(p), capacity) + } + } +} diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 18ca27a11a14..0d1b3139cb14 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -559,8 +559,8 @@ func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobF Hash: txs[i].Hash(), Tx: txs[i], Time: txs[i].Time(), - GasFeeCap: txs[i].GasFeeCap(), - GasTipCap: txs[i].GasTipCap(), + GasFeeCap: uint256.MustFromBig(txs[i].GasFeeCap()), + GasTipCap: uint256.MustFromBig(txs[i].GasTipCap()), Gas: txs[i].Gas(), BlobGas: txs[i].BlobGas(), } diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index aa19eef0d06a..edd15ec1ee70 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -35,9 +35,9 @@ type LazyTransaction struct { Hash common.Hash // Transaction hash to pull up if needed Tx *types.Transaction // Transaction if already resolved - Time time.Time // Time when the transaction was first seen - GasFeeCap *big.Int // Maximum fee per gas the transaction may consume - GasTipCap *big.Int // Maximum miner tip per gas the transaction can pay + Time time.Time // Time when the transaction was first seen + GasFeeCap *uint256.Int // Maximum fee per gas the transaction may consume + GasTipCap *uint256.Int // Maximum miner tip per gas the transaction can pay Gas uint64 // Amount of gas required by the transaction BlobGas uint64 // Amount of blob gas required by the transaction diff --git a/eth/handler_test.go b/eth/handler_test.go index 55f5c4486f16..0ca665156c95 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -112,8 +112,8 @@ func (p *testTxPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) diff --git a/miner/ordering.go b/miner/ordering.go index e686656bb2ba..c9ecb512f05d 100644 --- a/miner/ordering.go +++ b/miner/ordering.go @@ -21,28 +21,31 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" ) // txWithMinerFee wraps a transaction with its gas price or effective miner gasTipCap type txWithMinerFee struct { tx *txpool.LazyTransaction from common.Address - fees *big.Int + fees *uint256.Int } // newTxWithMinerFee creates a wrapped transaction, calculating the effective // miner gasTipCap if a base fee is provided. // Returns error in case of a negative effective miner gasTipCap. -func newTxWithMinerFee(tx *txpool.LazyTransaction, from common.Address, baseFee *big.Int) (*txWithMinerFee, error) { - tip := new(big.Int).Set(tx.GasTipCap) +func newTxWithMinerFee(tx *txpool.LazyTransaction, from common.Address, baseFee *uint256.Int) (*txWithMinerFee, error) { + tip := new(uint256.Int).Set(tx.GasTipCap) if baseFee != nil { if tx.GasFeeCap.Cmp(baseFee) < 0 { return nil, types.ErrGasFeeCapTooLow } - tip = math.BigMin(tx.GasTipCap, new(big.Int).Sub(tx.GasFeeCap, baseFee)) + tip = new(uint256.Int).Sub(tx.GasFeeCap, baseFee) + if tip.Gt(tx.GasTipCap) { + tip = tx.GasTipCap + } } return &txWithMinerFee{ tx: tx, @@ -87,7 +90,7 @@ type transactionsByPriceAndNonce struct { txs map[common.Address][]*txpool.LazyTransaction // Per account nonce-sorted list of transactions heads txByPriceAndTime // Next transaction for each unique account (price heap) signer types.Signer // Signer for the set of transactions - baseFee *big.Int // Current base fee + baseFee *uint256.Int // Current base fee } // newTransactionsByPriceAndNonce creates a transaction set that can retrieve @@ -96,10 +99,15 @@ type transactionsByPriceAndNonce struct { // Note, the input map is reowned so the caller should not interact any more with // if after providing it to the constructor. func newTransactionsByPriceAndNonce(signer types.Signer, txs map[common.Address][]*txpool.LazyTransaction, baseFee *big.Int) *transactionsByPriceAndNonce { + // Convert the basefee from header format to uint256 format + var baseFeeUint *uint256.Int + if baseFee != nil { + baseFeeUint = uint256.MustFromBig(baseFee) + } // Initialize a price and received time based heap with the head transactions heads := make(txByPriceAndTime, 0, len(txs)) for from, accTxs := range txs { - wrapped, err := newTxWithMinerFee(accTxs[0], from, baseFee) + wrapped, err := newTxWithMinerFee(accTxs[0], from, baseFeeUint) if err != nil { delete(txs, from) continue @@ -114,12 +122,12 @@ func newTransactionsByPriceAndNonce(signer types.Signer, txs map[common.Address] txs: txs, heads: heads, signer: signer, - baseFee: baseFee, + baseFee: baseFeeUint, } } // Peek returns the next transaction by price. -func (t *transactionsByPriceAndNonce) Peek() (*txpool.LazyTransaction, *big.Int) { +func (t *transactionsByPriceAndNonce) Peek() (*txpool.LazyTransaction, *uint256.Int) { if len(t.heads) == 0 { return nil, nil } diff --git a/miner/ordering_test.go b/miner/ordering_test.go index d2de9b9f3412..3587a835c884 100644 --- a/miner/ordering_test.go +++ b/miner/ordering_test.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/holiman/uint256" ) func TestTransactionPriceNonceSortLegacy(t *testing.T) { @@ -92,8 +93,8 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) @@ -160,8 +161,8 @@ func TestTransactionTimeSort(t *testing.T) { Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) diff --git a/miner/worker.go b/miner/worker.go index 6e4facdd0a94..c1726fc64b2d 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -206,7 +206,7 @@ type worker struct { mu sync.RWMutex // The lock used to protect the coinbase and extra fields coinbase common.Address extra []byte - tip *big.Int // Minimum tip needed for non-local transaction to include them + tip *uint256.Int // Minimum tip needed for non-local transaction to include them pendingMu sync.RWMutex pendingTasks map[common.Hash]*task @@ -253,7 +253,7 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus isLocalBlock: isLocalBlock, coinbase: config.Etherbase, extra: config.ExtraData, - tip: config.GasPrice, + tip: uint256.MustFromBig(config.GasPrice), pendingTasks: make(map[common.Hash]*task), txsCh: make(chan core.NewTxsEvent, txChanSize), chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), @@ -334,7 +334,7 @@ func (w *worker) setExtra(extra []byte) { func (w *worker) setGasTip(tip *big.Int) { w.mu.Lock() defer w.mu.Unlock() - w.tip = tip + w.tip = uint256.MustFromBig(tip) } // setRecommitInterval updates the interval for miner sealing work recommitting. @@ -556,15 +556,15 @@ func (w *worker) mainLoop() { Hash: tx.Hash(), Tx: nil, // Do *not* set this! We need to resolve it later to pull blobs in Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) } txset := newTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) tcount := w.current.tcount - w.commitTransactions(w.current, txset, nil, new(big.Int)) + w.commitTransactions(w.current, txset, nil, new(uint256.Int)) // Only update the snapshot if any new transactions were added // to the pending block @@ -802,7 +802,7 @@ func (w *worker) applyTransaction(env *environment, tx *types.Transaction) (*typ return receipt, err } -func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAndNonce, interrupt *atomic.Int32, minTip *big.Int) error { +func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAndNonce, interrupt *atomic.Int32, minTip *uint256.Int) error { gasLimit := env.header.GasLimit if env.gasPool == nil { env.gasPool = new(core.GasPool).AddGas(gasLimit) @@ -1013,7 +1013,7 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err if env.header.ExcessBlobGas != nil { blobFee = uint256.MustFromBig(eip4844.CalcBlobFee(*env.header.ExcessBlobGas)) } - pending := w.eth.TxPool().Pending(uint256.MustFromBig(tip), baseFee, blobFee) + pending := w.eth.TxPool().Pending(tip, baseFee, blobFee) // Split the pending transactions into locals and remotes. localTxs, remoteTxs := make(map[common.Address][]*txpool.LazyTransaction), pending @@ -1027,7 +1027,7 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err // Fill the block with all available pending transactions. if len(localTxs) > 0 { txs := newTransactionsByPriceAndNonce(env.signer, localTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt, new(big.Int)); err != nil { + if err := w.commitTransactions(env, txs, interrupt, new(uint256.Int)); err != nil { return err } } From ac0ff044606a663eeb47ef60ed5506f842753084 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Mon, 19 Feb 2024 16:29:59 +0100 Subject: [PATCH 03/91] core/vm, params: ensure order of forks, prevent overflow (#29023) This PR fixes an overflow which can could happen if inconsistent blockchain rules were configured. Additionally, it tries to prevent such inconsistencies from occurring by making sure that merge cannot be enabled unless previous fork(s) are also enabled. --- core/vm/operations_acl.go | 7 ++++++- internal/ethapi/api_test.go | 2 +- params/config.go | 10 ++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index bca6d1e83b88..f420a241058b 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -187,7 +187,12 @@ func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc { // outside of this function, as part of the dynamic gas, and that will make it // also become correctly reported to tracers. contract.Gas += coldCost - return gas + coldCost, nil + + var overflow bool + if gas, overflow = math.SafeAdd(gas, coldCost); overflow { + return 0, ErrGasUintOverflow + } + return gas, nil } } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 8a2e367f4a83..a6f7405eb363 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -1818,6 +1818,7 @@ func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Ha tx *types.Transaction err error ) + b.SetPoS() switch i { case 0: // transfer 1000wei @@ -1866,7 +1867,6 @@ func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Ha b.AddTx(tx) txHashes[i] = tx.Hash() } - b.SetPoS() }) return backend, txHashes } diff --git a/params/config.go b/params/config.go index 2c80f4f6b09b..d6935ed70cf7 100644 --- a/params/config.go +++ b/params/config.go @@ -910,6 +910,8 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules if chainID == nil { chainID = new(big.Int) } + // disallow setting Merge out of order + isMerge = isMerge && c.IsLondon(num) return Rules{ ChainID: new(big.Int).Set(chainID), IsHomestead: c.IsHomestead(num), @@ -923,9 +925,9 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules IsBerlin: c.IsBerlin(num), IsLondon: c.IsLondon(num), IsMerge: isMerge, - IsShanghai: c.IsShanghai(num, timestamp), - IsCancun: c.IsCancun(num, timestamp), - IsPrague: c.IsPrague(num, timestamp), - IsVerkle: c.IsVerkle(num, timestamp), + IsShanghai: isMerge && c.IsShanghai(num, timestamp), + IsCancun: isMerge && c.IsCancun(num, timestamp), + IsPrague: isMerge && c.IsPrague(num, timestamp), + IsVerkle: isMerge && c.IsVerkle(num, timestamp), } } From f4852b8ddc8bef962d34210a4f7774b95767e421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 20 Feb 2024 11:37:23 +0200 Subject: [PATCH 04/91] core/txpool, eth, miner: retrieve plain and blob txs separately (#29026) * core/txpool, eth, miner: retrieve plain and blob txs separately * core/txpool: fix typo, no farming * miner: farm all the typos Co-authored-by: Martin HS --------- Co-authored-by: Martin HS --- core/txpool/blobpool/blobpool.go | 19 +++--- core/txpool/blobpool/blobpool_test.go | 6 +- core/txpool/legacypool/legacypool.go | 16 +++-- core/txpool/subpool.go | 17 +++++- core/txpool/txpool.go | 5 +- eth/api_backend.go | 2 +- eth/catalyst/simulated_beacon.go | 5 +- eth/handler.go | 3 +- eth/handler_test.go | 2 +- eth/sync.go | 3 +- miner/ordering.go | 11 ++++ miner/worker.go | 86 +++++++++++++++++++-------- 12 files changed, 125 insertions(+), 50 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 0ab382001a10..d1fe7a60640d 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1446,7 +1446,12 @@ func (p *BlobPool) drop() { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction { +func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { + // If only plain transactions are requested, this pool is unsuitable as it + // contains none, don't even bother. + if filter.OnlyPlainTxs { + return nil + } // Track the amount of time waiting to retrieve the list of pending blob txs // from the pool and the amount of time actually spent on assembling the data. // The latter will be pretty much moot, but we've kept it to have symmetric @@ -1466,20 +1471,20 @@ func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *u lazies := make([]*txpool.LazyTransaction, 0, len(txs)) for _, tx := range txs { // If transaction filtering was requested, discard badly priced ones - if minTip != nil && baseFee != nil { - if tx.execFeeCap.Lt(baseFee) { + if filter.MinTip != nil && filter.BaseFee != nil { + if tx.execFeeCap.Lt(filter.BaseFee) { break // basefee too low, cannot be included, discard rest of txs from the account } - tip := new(uint256.Int).Sub(tx.execFeeCap, baseFee) + tip := new(uint256.Int).Sub(tx.execFeeCap, filter.BaseFee) if tip.Gt(tx.execTipCap) { tip = tx.execTipCap } - if tip.Lt(minTip) { + if tip.Lt(filter.MinTip) { break // allowed or remaining tip too low, cannot be included, discard rest of txs from the account } } - if blobFee != nil { - if tx.blobFeeCap.Lt(blobFee) { + if filter.BlobFee != nil { + if tx.blobFeeCap.Lt(filter.BlobFee) { break // blobfee too low, cannot be included, discard rest of txs from the account } } diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 4cec78b57263..579d42a2dcdf 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -1340,7 +1340,11 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) { b.ReportAllocs() for i := 0; i < b.N; i++ { - p := pool.Pending(uint256.NewInt(1), chain.basefee, chain.blobfee) + p := pool.Pending(txpool.PendingFilter{ + MinTip: uint256.NewInt(1), + BaseFee: chain.basefee, + BlobFee: chain.blobfee, + }) if len(p) != int(capacity) { b.Fatalf("have %d want %d", len(p), capacity) } diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 0d1b3139cb14..8e7095f296a6 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -522,7 +522,12 @@ func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction, // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction { +func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { + // If only blob transactions are requested, this pool is unsuitable as it + // contains none, don't even bother. + if filter.OnlyBlobTxs { + return nil + } pool.mu.Lock() defer pool.mu.Unlock() @@ -531,13 +536,12 @@ func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobF minTipBig *big.Int baseFeeBig *big.Int ) - if minTip != nil { - minTipBig = minTip.ToBig() + if filter.MinTip != nil { + minTipBig = filter.MinTip.ToBig() } - if baseFee != nil { - baseFeeBig = baseFee.ToBig() + if filter.BaseFee != nil { + baseFeeBig = filter.BaseFee.ToBig() } - pending := make(map[common.Address][]*txpool.LazyTransaction, len(pool.pending)) for addr, list := range pool.pending { txs := list.Flatten() diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index edd15ec1ee70..9881ed1b8f96 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -70,6 +70,21 @@ type LazyResolver interface { // may request (and relinquish) exclusive access to certain addresses. type AddressReserver func(addr common.Address, reserve bool) error +// PendingFilter is a collection of filter rules to allow retrieving a subset +// of transactions for announcement or mining. +// +// Note, the entries here are not arbitrary useful filters, rather each one has +// a very specific call site in mind and each one can be evaluated very cheaply +// by the pool implementations. Only add new ones that satisfy those constraints. +type PendingFilter struct { + MinTip *uint256.Int // Minimum miner tip required to include a transaction + BaseFee *uint256.Int // Minimum 1559 basefee needed to include a transaction + BlobFee *uint256.Int // Minimum 4844 blobfee needed to include a blob transaction + + OnlyPlainTxs bool // Return only plain EVM transactions (peer-join announces, block space filling) + OnlyBlobTxs bool // Return only blob transactions (block blob-space filling) +} + // SubPool represents a specialized transaction pool that lives on its own (e.g. // blob pool). Since independent of how many specialized pools we have, they do // need to be updated in lockstep and assemble into one coherent view for block @@ -118,7 +133,7 @@ type SubPool interface { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. - Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*LazyTransaction + Pending(filter PendingFilter) map[common.Address][]*LazyTransaction // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 3d0d6bf617bd..8bf3e0a51261 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "github.com/holiman/uint256" ) // TxStatus is the current status of a transaction as seen by the pool. @@ -357,10 +356,10 @@ func (p *TxPool) Add(txs []*types.Transaction, local bool, sync bool) []error { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (p *TxPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*LazyTransaction { +func (p *TxPool) Pending(filter PendingFilter) map[common.Address][]*LazyTransaction { txs := make(map[common.Address][]*LazyTransaction) for _, subpool := range p.subpools { - for addr, set := range subpool.Pending(minTip, baseFee, blobFee) { + for addr, set := range subpool.Pending(filter) { txs[addr] = set } } diff --git a/eth/api_backend.go b/eth/api_backend.go index c24fa313936f..65adccd8518c 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -292,7 +292,7 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { - pending := b.eth.txPool.Pending(nil, nil, nil) + pending := b.eth.txPool.Pending(txpool.PendingFilter{}) var txs types.Transactions for _, batch := range pending { for _, lazy := range batch { diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index 91ac1771d283..f1c5689e1d28 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/log" @@ -263,7 +264,7 @@ func (c *SimulatedBeacon) Rollback() { // Fork sets the head to the provided hash. func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { - if len(c.eth.TxPool().Pending(nil, nil, nil)) != 0 { + if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { return errors.New("pending block dirty") } parent := c.eth.BlockChain().GetBlockByHash(parentHash) @@ -275,7 +276,7 @@ func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { // AdjustTime creates a new block with an adjusted timestamp. func (c *SimulatedBeacon) AdjustTime(adjustment time.Duration) error { - if len(c.eth.TxPool().Pending(nil, nil, nil)) != 0 { + if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { return errors.New("could not adjust time on non-empty block") } parent := c.eth.BlockChain().CurrentBlock() diff --git a/eth/handler.go b/eth/handler.go index b2fef62ea3c4..0343a5787012 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -42,7 +42,6 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/triedb/pathdb" - "github.com/holiman/uint256" ) const ( @@ -74,7 +73,7 @@ type txPool interface { // Pending should return pending transactions. // The slice should be modifiable by the caller. - Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction + Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions diff --git a/eth/handler_test.go b/eth/handler_test.go index 0ca665156c95..58353f6b6452 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -93,7 +93,7 @@ func (p *testTxPool) Add(txs []*types.Transaction, local bool, sync bool) []erro } // Pending returns all the transactions known to the pool -func (p *testTxPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction { +func (p *testTxPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { p.lock.RLock() defer p.lock.RUnlock() diff --git a/eth/sync.go b/eth/sync.go index fa3a4088043e..cdcfbdb3db49 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" @@ -36,7 +37,7 @@ const ( // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { var hashes []common.Hash - for _, batch := range h.txpool.Pending(nil, nil, nil) { + for _, batch := range h.txpool.Pending(txpool.PendingFilter{OnlyPlainTxs: true}) { for _, tx := range batch { hashes = append(hashes, tx.Hash) } diff --git a/miner/ordering.go b/miner/ordering.go index c9ecb512f05d..bcf7af46e891 100644 --- a/miner/ordering.go +++ b/miner/ordering.go @@ -153,3 +153,14 @@ func (t *transactionsByPriceAndNonce) Shift() { func (t *transactionsByPriceAndNonce) Pop() { heap.Pop(&t.heads) } + +// Empty returns if the price heap is empty. It can be used to check it simpler +// than calling peek and checking for nil return. +func (t *transactionsByPriceAndNonce) Empty() bool { + return len(t.heads) == 0 +} + +// Clear removes the entire content of the heap. +func (t *transactionsByPriceAndNonce) Clear() { + t.heads, t.txs = nil, nil +} diff --git a/miner/worker.go b/miner/worker.go index c1726fc64b2d..9a36106231e9 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -562,9 +562,11 @@ func (w *worker) mainLoop() { BlobGas: tx.BlobGas(), }) } - txset := newTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) + plainTxs := newTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) // Mixed bag of everrything, yolo + blobTxs := newTransactionsByPriceAndNonce(w.current.signer, nil, w.current.header.BaseFee) // Empty bag, don't bother optimising + tcount := w.current.tcount - w.commitTransactions(w.current, txset, nil, new(uint256.Int)) + w.commitTransactions(w.current, plainTxs, blobTxs, nil) // Only update the snapshot if any new transactions were added // to the pending block @@ -802,7 +804,7 @@ func (w *worker) applyTransaction(env *environment, tx *types.Transaction) (*typ return receipt, err } -func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAndNonce, interrupt *atomic.Int32, minTip *uint256.Int) error { +func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { gasLimit := env.header.GasLimit if env.gasPool == nil { env.gasPool = new(core.GasPool).AddGas(gasLimit) @@ -821,8 +823,33 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) break } + // If we don't have enough blob space for any further blob transactions, + // skip that list altogether + if !blobTxs.Empty() && env.blobs*params.BlobTxBlobGasPerBlob >= params.MaxBlobGasPerBlock { + log.Trace("Not enough blob space for further blob transactions") + blobTxs.Clear() + // Fall though to pick up any plain txs + } // Retrieve the next transaction and abort if all done. - ltx, tip := txs.Peek() + var ( + ltx *txpool.LazyTransaction + txs *transactionsByPriceAndNonce + ) + pltx, ptip := plainTxs.Peek() + bltx, btip := blobTxs.Peek() + + switch { + case pltx == nil: + txs, ltx = blobTxs, bltx + case bltx == nil: + txs, ltx = plainTxs, pltx + default: + if ptip.Lt(btip) { + txs, ltx = blobTxs, bltx + } else { + txs, ltx = plainTxs, pltx + } + } if ltx == nil { break } @@ -837,11 +864,6 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn txs.Pop() continue } - // If we don't receive enough tip for the next transaction, skip the account - if tip.Cmp(minTip) < 0 { - log.Trace("Not enough tip for transaction", "hash", ltx.Hash, "tip", tip, "needed", minTip) - break // If the next-best is too low, surely no better will be available - } // Transaction seems to fit, pull it up from the pool tx := ltx.Resolve() if tx == nil { @@ -1005,35 +1027,49 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err w.mu.RUnlock() // Retrieve the pending transactions pre-filtered by the 1559/4844 dynamic fees - var baseFee *uint256.Int + filter := txpool.PendingFilter{ + MinTip: tip, + } if env.header.BaseFee != nil { - baseFee = uint256.MustFromBig(env.header.BaseFee) + filter.BaseFee = uint256.MustFromBig(env.header.BaseFee) } - var blobFee *uint256.Int if env.header.ExcessBlobGas != nil { - blobFee = uint256.MustFromBig(eip4844.CalcBlobFee(*env.header.ExcessBlobGas)) + filter.BlobFee = uint256.MustFromBig(eip4844.CalcBlobFee(*env.header.ExcessBlobGas)) } - pending := w.eth.TxPool().Pending(tip, baseFee, blobFee) + filter.OnlyPlainTxs, filter.OnlyBlobTxs = true, false + pendingPlainTxs := w.eth.TxPool().Pending(filter) + + filter.OnlyPlainTxs, filter.OnlyBlobTxs = false, true + pendingBlobTxs := w.eth.TxPool().Pending(filter) // Split the pending transactions into locals and remotes. - localTxs, remoteTxs := make(map[common.Address][]*txpool.LazyTransaction), pending + localPlainTxs, remotePlainTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingPlainTxs + localBlobTxs, remoteBlobTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingBlobTxs + for _, account := range w.eth.TxPool().Locals() { - if txs := remoteTxs[account]; len(txs) > 0 { - delete(remoteTxs, account) - localTxs[account] = txs + if txs := remotePlainTxs[account]; len(txs) > 0 { + delete(remotePlainTxs, account) + localPlainTxs[account] = txs + } + if txs := remoteBlobTxs[account]; len(txs) > 0 { + delete(remoteBlobTxs, account) + localBlobTxs[account] = txs } } - // Fill the block with all available pending transactions. - if len(localTxs) > 0 { - txs := newTransactionsByPriceAndNonce(env.signer, localTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt, new(uint256.Int)); err != nil { + if len(localPlainTxs) > 0 || len(localBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, localPlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, localBlobTxs, env.header.BaseFee) + + if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } - if len(remoteTxs) > 0 { - txs := newTransactionsByPriceAndNonce(env.signer, remoteTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt, tip); err != nil { + if len(remotePlainTxs) > 0 || len(remoteBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, remotePlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, remoteBlobTxs, env.header.BaseFee) + + if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } From 7f5e96dc6c0d70f793a6a41c059c5dd660357964 Mon Sep 17 00:00:00 2001 From: buddho Date: Tue, 20 Feb 2024 18:08:56 +0800 Subject: [PATCH 05/91] core/txpool: fix typo (#29031) --- core/txpool/blobpool/blobpool.go | 2 +- core/txpool/blobpool/blobpool_test.go | 4 ++-- core/txpool/blobpool/evictheap.go | 2 +- core/txpool/blobpool/priority_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index d1fe7a60640d..fcd520603ffc 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -371,7 +371,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres } p.head, p.state = head, state - // Index all transactions on disk and delete anything inprocessable + // Index all transactions on disk and delete anything unprocessable var fails []uint64 index := func(id uint64, size uint32, blob []byte) { if p.parseTransaction(id, size, blob) != nil { diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 579d42a2dcdf..be5833011a07 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -185,7 +185,7 @@ func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64, return types.MustSignNewTx(key, types.LatestSigner(testChainConfig), blobtx) } -// makeUnsignedTx is a utility method to construct a random blob tranasaction +// makeUnsignedTx is a utility method to construct a random blob transaction // without signing it. func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64) *types.BlobTx { return &types.BlobTx{ @@ -391,7 +391,7 @@ func TestOpenDrops(t *testing.T) { id, _ := store.Put(blob) filled[id] = struct{}{} } - // Insert a sequence of transactions with partially passed nonces to veirfy + // Insert a sequence of transactions with partially passed nonces to verify // that the included part of the set will get dropped (case 4). var ( overlapper, _ = crypto.GenerateKey() diff --git a/core/txpool/blobpool/evictheap.go b/core/txpool/blobpool/evictheap.go index df594099f79b..bc4543a352e2 100644 --- a/core/txpool/blobpool/evictheap.go +++ b/core/txpool/blobpool/evictheap.go @@ -30,7 +30,7 @@ import ( // transaction from each account to determine which account to evict from. // // The heap internally tracks a slice of cheapest transactions from each account -// and a mapping from addresses to indices for direct removals/udates. +// and a mapping from addresses to indices for direct removals/updates. // // The goal of the heap is to decide which account has the worst bottleneck to // evict transactions from. diff --git a/core/txpool/blobpool/priority_test.go b/core/txpool/blobpool/priority_test.go index 4aad919925f5..cf0e0454a00a 100644 --- a/core/txpool/blobpool/priority_test.go +++ b/core/txpool/blobpool/priority_test.go @@ -64,7 +64,7 @@ func BenchmarkDynamicFeeJumpCalculation(b *testing.B) { // Benchmarks how many priority recalculations can be done. func BenchmarkPriorityCalculation(b *testing.B) { // The basefee and blob fee is constant for all transactions across a block, - // so we can assume theit absolute jump counts can be pre-computed. + // so we can assume their absolute jump counts can be pre-computed. basefee := uint256.NewInt(17_200_000_000) // 17.2 Gwei is the 22.03.2023 zero-emission basefee, random number blobfee := uint256.NewInt(123_456_789_000) // Completely random, no idea what this will be From bba3fa9af9709ce6615d994edac7043e064fda0d Mon Sep 17 00:00:00 2001 From: buddho Date: Tue, 20 Feb 2024 19:42:48 +0800 Subject: [PATCH 06/91] core,eth,internal: fix typo (#29024) --- core/state/sync.go | 2 +- core/txpool/legacypool/list.go | 2 +- eth/api_miner.go | 2 +- eth/downloader/api.go | 2 +- eth/protocols/eth/peer.go | 2 +- eth/protocols/snap/peer.go | 4 ++-- internal/ethapi/api.go | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/state/sync.go b/core/state/sync.go index d6775e889610..411b54eab096 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// NewStateSync create a new state trie download scheduler. +// NewStateSync creates a new state trie download scheduler. func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync { // Register the storage slot callback if the external callback is specified. var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error diff --git a/core/txpool/legacypool/list.go b/core/txpool/legacypool/list.go index f0f9f213f27d..7db9c98ace63 100644 --- a/core/txpool/legacypool/list.go +++ b/core/txpool/legacypool/list.go @@ -278,7 +278,7 @@ type list struct { totalcost *uint256.Int // Total cost of all transactions in the list } -// newList create a new transaction list for maintaining nonce-indexable fast, +// newList creates a new transaction list for maintaining nonce-indexable fast, // gapped, sortable transaction lists. func newList(strict bool) *list { return &list{ diff --git a/eth/api_miner.go b/eth/api_miner.go index 2fe296548a20..764d0ae5e2f5 100644 --- a/eth/api_miner.go +++ b/eth/api_miner.go @@ -29,7 +29,7 @@ type MinerAPI struct { e *Ethereum } -// NewMinerAPI create a new MinerAPI instance. +// NewMinerAPI creates a new MinerAPI instance. func NewMinerAPI(e *Ethereum) *MinerAPI { return &MinerAPI{e} } diff --git a/eth/downloader/api.go b/eth/downloader/api.go index f09122904c4c..6b8cb98e23bd 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -38,7 +38,7 @@ type DownloaderAPI struct { uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest } -// NewDownloaderAPI create a new DownloaderAPI. The API has an internal event loop that +// NewDownloaderAPI creates a new DownloaderAPI. The API has an internal event loop that // listens for events from the downloader through the global event mux. In case it receives one of // these events it broadcasts it to all syncing subscriptions that are installed through the // installSyncSubscription channel. diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index caa5239cf98a..ffd78b05946a 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -92,7 +92,7 @@ type Peer struct { lock sync.RWMutex // Mutex protecting the internal fields } -// NewPeer create a wrapper for a network connection and negotiated protocol +// NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Peer { peer := &Peer{ diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index 3db6e22cbd92..c57931678cec 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -33,7 +33,7 @@ type Peer struct { logger log.Logger // Contextual logger with the peer id injected } -// NewPeer create a wrapper for a network connection and negotiated protocol +// NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { id := p.ID().String() @@ -46,7 +46,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { } } -// NewFakePeer create a fake snap peer without a backing p2p peer, for testing purposes. +// NewFakePeer creates a fake snap peer without a backing p2p peer, for testing purposes. func NewFakePeer(version uint, id string, rw p2p.MsgReadWriter) *Peer { return &Peer{ id: id, diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index df25dfbd37a6..e594154daa4b 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -288,7 +288,7 @@ type PersonalAccountAPI struct { b Backend } -// NewPersonalAccountAPI create a new PersonalAccountAPI. +// NewPersonalAccountAPI creates a new PersonalAccountAPI. func NewPersonalAccountAPI(b Backend, nonceLock *AddrLocker) *PersonalAccountAPI { return &PersonalAccountAPI{ am: b.AccountManager(), From 79e340fb1276cd5f0bbdc3825f90090488e3b978 Mon Sep 17 00:00:00 2001 From: Haotian <51777534+tmelhao@users.noreply.github.com> Date: Wed, 21 Feb 2024 15:59:21 +0800 Subject: [PATCH 07/91] params: add cancun upgrade banner (#29042) params: add cancun banner Signed-off-by: tmelhao Co-authored-by: tmelhao --- params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/config.go b/params/config.go index d6935ed70cf7..21ede457fd68 100644 --- a/params/config.go +++ b/params/config.go @@ -467,7 +467,7 @@ func (c *ChainConfig) Description() string { banner += fmt.Sprintf(" - Shanghai: @%-10v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/shanghai.md)\n", *c.ShanghaiTime) } if c.CancunTime != nil { - banner += fmt.Sprintf(" - Cancun: @%-10v\n", *c.CancunTime) + banner += fmt.Sprintf(" - Cancun: @%-10v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/cancun.md)\n", *c.CancunTime) } if c.PragueTime != nil { banner += fmt.Sprintf(" - Prague: @%-10v\n", *c.PragueTime) From b9ca38b7358dbf7e236c624043bbab789a8d0389 Mon Sep 17 00:00:00 2001 From: colin <102356659+colinlyguo@users.noreply.github.com> Date: Wed, 21 Feb 2024 16:00:01 +0800 Subject: [PATCH 08/91] core/txpool: fix typo (#29036) * fix typos * address comments --- core/state_transition.go | 2 +- core/txpool/blobpool/blobpool.go | 12 ++++++------ core/txpool/legacypool/legacypool.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index 2be54480f393..9c4f76d1c585 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -67,7 +67,7 @@ func (result *ExecutionResult) Revert() []byte { } // IntrinsicGas computes the 'intrinsic gas' for a message with the given data. -func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028 bool, isEIP3860 bool) (uint64, error) { +func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028, isEIP3860 bool) (uint64, error) { // Set the starting gas for the raw transaction var gas uint64 if isContractCreation && isHomestead { diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index fcd520603ffc..276c2886e2dc 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -360,7 +360,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres } } // Initialize the state with head block, or fallback to empty one in - // case the head state is not available(might occur when node is not + // case the head state is not available (might occur when node is not // fully synced). state, err := p.chain.StateAt(head.Root) if err != nil { @@ -540,7 +540,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 } delete(p.index, addr) delete(p.spent, addr) - if inclusions != nil { // only during reorgs will the heap will be initialized + if inclusions != nil { // only during reorgs will the heap be initialized heap.Remove(p.evict, p.evict.index[addr]) } p.reserve(addr, false) @@ -693,7 +693,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 if len(txs) == 0 { delete(p.index, addr) delete(p.spent, addr) - if inclusions != nil { // only during reorgs will the heap will be initialized + if inclusions != nil { // only during reorgs will the heap be initialized heap.Remove(p.evict, p.evict.index[addr]) } p.reserve(addr, false) @@ -809,7 +809,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { } } // Recheck the account's pooled transactions to drop included and - // invalidated one + // invalidated ones p.recheck(addr, inclusions) } if len(adds) > 0 { @@ -1226,7 +1226,7 @@ func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error // consensus validity and pool restrictions). func (p *BlobPool) add(tx *types.Transaction) (err error) { // The blob pool blocks on adding a transaction. This is because blob txs are - // only even pulled form the network, so this method will act as the overload + // only even pulled from the network, so this method will act as the overload // protection for fetches. waitStart := time.Now() p.lock.Lock() @@ -1554,7 +1554,7 @@ func (p *BlobPool) updateStorageMetrics() { } // updateLimboMetrics retrieves a bunch of stats from the limbo store and pushes -// // them out as metrics. +// them out as metrics. func (p *BlobPool) updateLimboMetrics() { stats := p.limbo.store.Infos() diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 8e7095f296a6..4e1d26acf405 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -296,7 +296,7 @@ func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserve txpool.A pool.gasTip.Store(uint256.NewInt(gasTip)) // Initialize the state with head block, or fallback to empty one in - // case the head state is not available(might occur when node is not + // case the head state is not available (might occur when node is not // fully synced). statedb, err := pool.chain.StateAt(head.Root) if err != nil { From b47cf8fe1de4f97ce38417d8136a58812734a7a9 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Wed, 21 Feb 2024 12:46:32 +0100 Subject: [PATCH 09/91] internal/ethapi: fix defaults for blob fields (#29037) Co-authored-by: Martin HS --- internal/ethapi/transaction_args.go | 36 +++++++++++----------- internal/ethapi/transaction_args_test.go | 38 +++++++++++------------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 03ffb7524f59..d221c14db517 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -177,6 +177,14 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { // setFeeDefaults fills in default fee values for unspecified tx fields. func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) error { + head := b.CurrentHeader() + // Sanity check the EIP-4844 fee parameters. + if args.BlobFeeCap != nil && args.BlobFeeCap.ToInt().Sign() == 0 { + return errors.New("maxFeePerBlobGas, if specified, must be non-zero") + } + if err := args.setCancunFeeDefaults(ctx, head, b); err != nil { + return err + } // If both gasPrice and at least one of the EIP-1559 fee parameters are specified, error. if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") @@ -186,7 +194,6 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro // other tx values. See https://github.com/ethereum/go-ethereum/pull/23274 // for more information. eip1559ParamsSet := args.MaxFeePerGas != nil && args.MaxPriorityFeePerGas != nil - // Sanity check the EIP-1559 fee parameters if present. if args.GasPrice == nil && eip1559ParamsSet { if args.MaxFeePerGas.ToInt().Sign() == 0 { @@ -198,13 +205,7 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro return nil // No need to set anything, user already set MaxFeePerGas and MaxPriorityFeePerGas } - // Sanity check the EIP-4844 fee parameters. - if args.BlobFeeCap != nil && args.BlobFeeCap.ToInt().Sign() == 0 { - return errors.New("maxFeePerBlobGas must be non-zero") - } - // Sanity check the non-EIP-1559 fee parameters. - head := b.CurrentHeader() isLondon := b.ChainConfig().IsLondon(head.Number) if args.GasPrice != nil && !eip1559ParamsSet { // Zero gas-price is not allowed after London fork @@ -215,21 +216,14 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro } // Now attempt to fill in default value depending on whether London is active or not. - if b.ChainConfig().IsCancun(head.Number, head.Time) { - if err := args.setCancunFeeDefaults(ctx, head, b); err != nil { - return err - } - } else if isLondon { - if args.BlobFeeCap != nil { - return errors.New("maxFeePerBlobGas is not valid before Cancun is active") - } + if isLondon { // London is active, set maxPriorityFeePerGas and maxFeePerGas. if err := args.setLondonFeeDefaults(ctx, head, b); err != nil { return err } } else { - if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil || args.BlobFeeCap != nil { - return errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active") + if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { + return errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active") } // London not active, set gas price. price, err := b.SuggestGasTipCap(ctx) @@ -245,15 +239,19 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro func (args *TransactionArgs) setCancunFeeDefaults(ctx context.Context, head *types.Header, b Backend) error { // Set maxFeePerBlobGas if it is missing. if args.BlobHashes != nil && args.BlobFeeCap == nil { + var excessBlobGas uint64 + if head.ExcessBlobGas != nil { + excessBlobGas = *head.ExcessBlobGas + } // ExcessBlobGas must be set for a Cancun block. - blobBaseFee := eip4844.CalcBlobFee(*head.ExcessBlobGas) + blobBaseFee := eip4844.CalcBlobFee(excessBlobGas) // Set the max fee to be 2 times larger than the previous block's blob base fee. // The additional slack allows the tx to not become invalidated if the base // fee is rising. val := new(big.Int).Mul(blobBaseFee, big.NewInt(2)) args.BlobFeeCap = (*hexutil.Big)(val) } - return args.setLondonFeeDefaults(ctx, head, b) + return nil } // setLondonFeeDefaults fills in reasonable default fee values for unspecified fields. diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index f0fdb6d8ee2d..1b1634b25031 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -153,14 +153,14 @@ func TestSetFeeDefaults(t *testing.T) { "legacy", &TransactionArgs{MaxFeePerGas: maxFee}, nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), + errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), }, { "dynamic fee tx pre-London, priorityFee set", "legacy", &TransactionArgs{MaxPriorityFeePerGas: fortytwo}, nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), + errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), }, { "dynamic fee tx, maxFee < priorityFee", @@ -207,20 +207,6 @@ func TestSetFeeDefaults(t *testing.T) { errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), }, // EIP-4844 - { - "set maxFeePerBlobGas pre cancun", - "london", - &TransactionArgs{BlobFeeCap: fortytwo}, - nil, - errors.New("maxFeePerBlobGas is not valid before Cancun is active"), - }, - { - "set maxFeePerBlobGas pre london", - "legacy", - &TransactionArgs{BlobFeeCap: fortytwo}, - nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), - }, { "set gas price and maxFee for blob transaction", "cancun", @@ -235,6 +221,13 @@ func TestSetFeeDefaults(t *testing.T) { &TransactionArgs{BlobHashes: []common.Hash{}, BlobFeeCap: (*hexutil.Big)(big.NewInt(4)), MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, nil, }, + { + "fill maxFeePerBlobGas when dynamic fees are set", + "cancun", + &TransactionArgs{BlobHashes: []common.Hash{}, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + &TransactionArgs{BlobHashes: []common.Hash{}, BlobFeeCap: (*hexutil.Big)(big.NewInt(4)), MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, } ctx := context.Background() @@ -244,11 +237,16 @@ func TestSetFeeDefaults(t *testing.T) { } got := test.in err := got.setFeeDefaults(ctx, b) - if err != nil && err.Error() == test.err.Error() { - // Test threw expected error. + if err != nil { + if test.err == nil { + t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } else if err.Error() != test.err.Error() { + t.Fatalf("test %d (%s): unexpected error: (got: %s, want: %s)", i, test.name, err, test.err) + } + // Matching error. continue - } else if err != nil { - t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } else if test.err != nil { + t.Fatalf("test %d (%s): expected error: %s", i, test.name, test.err) } if !reflect.DeepEqual(got, test.want) { t.Fatalf("test %d (%s): did not fill defaults as expected: (got: %v, want: %v)", i, test.name, got, test.want) From 3b4ede74443a15db27fddbb803a6b0cc4180ca75 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 21 Feb 2024 15:44:02 +0100 Subject: [PATCH 10/91] params: release go-ethereum v1.13.13 stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 7284c07524f7..19b22e029cb1 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 13 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 13 // Minor version component of the current release + VersionPatch = 13 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From b590cae89232299d54aac8aada88c66d00c5b34c Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 21 Feb 2024 15:49:50 +0100 Subject: [PATCH 11/91] params: begin v1.13.14 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 19b22e029cb1..34ba3f74200b 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 13 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 13 // Minor version component of the current release + VersionPatch = 14 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From e47a7c22c40b9037049cb63d74eb1216aabdee60 Mon Sep 17 00:00:00 2001 From: ArtificialPB Date: Thu, 22 Feb 2024 14:39:22 +0100 Subject: [PATCH 12/91] internal/ethapi: use overriden baseFee for gasPrice (#29051) eth_call and debug_traceCall allow users to override various block fields, among them base fee. However the overriden base fee was not considered for computing the effective gas price of that message, and instead base fee of the base block was used. This has been fixed in this commit. --- eth/tracers/api.go | 2 +- internal/ethapi/api.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 4d4428f6c63b..683310820526 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -919,7 +919,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc config.BlockOverrides.Apply(&vmctx) } // Execute the trace - msg, err := args.ToMessage(api.backend.RPCGasCap(), block.BaseFee()) + msg, err := args.ToMessage(api.backend.RPCGasCap(), vmctx.BaseFee) if err != nil { return nil, err } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index e594154daa4b..02aeaff0c680 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1093,14 +1093,14 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S defer cancel() // Get a new instance of the EVM. - msg, err := args.ToMessage(globalGasCap, header.BaseFee) - if err != nil { - return nil, err - } blockCtx := core.NewEVMBlockContext(header, NewChainContext(ctx, b), nil) if blockOverrides != nil { blockOverrides.Apply(&blockCtx) } + msg, err := args.ToMessage(globalGasCap, blockCtx.BaseFee) + if err != nil { + return nil, err + } evm := b.GetEVM(ctx, msg, state, header, &vm.Config{NoBaseFee: true}, &blockCtx) // Wait for the context to be done and cancel the evm. Even if the From b87b9b45331f87fb1da379c5f17a81ebc3738c6e Mon Sep 17 00:00:00 2001 From: colin <102356659+colinlyguo@users.noreply.github.com> Date: Thu, 22 Feb 2024 23:35:23 +0800 Subject: [PATCH 13/91] internal/ethapi:fix zero rpc gas cap in eth_createAccessList (#28846) This PR enhances eth_createAccessList RPC call to support scenarios where the node is launched with an unlimited gas cap (--rpc.gascap 0). The eth_createAccessList RPC call returns failure if user doesn't explicitly set a gas limit. --- internal/ethapi/api.go | 17 ++++------ internal/ethapi/transaction_args.go | 51 ++++++++++++++++------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 02aeaff0c680..863849f4da6a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -453,7 +453,7 @@ func (s *PersonalAccountAPI) signTransaction(ctx context.Context, args *Transact return nil, err } // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Assemble the transaction and sign with the wallet @@ -1485,14 +1485,9 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if db == nil || err != nil { return nil, 0, nil, err } - // If the gas amount is not set, default to RPC gas cap. - if args.Gas == nil { - tmp := hexutil.Uint64(b.RPCGasCap()) - args.Gas = &tmp - } // Ensure any missing fields are filled, extract the recipient and input data - if err := args.setDefaults(ctx, b); err != nil { + if err := args.setDefaults(ctx, b, true); err != nil { return nil, 0, nil, err } var to common.Address @@ -1795,7 +1790,7 @@ func (s *TransactionAPI) SendTransaction(ctx context.Context, args TransactionAr } // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return common.Hash{}, err } // Assemble the transaction and sign with the wallet @@ -1815,7 +1810,7 @@ func (s *TransactionAPI) FillTransaction(ctx context.Context, args TransactionAr args.blobSidecarAllowed = true // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Assemble the transaction and obtain rlp @@ -1884,7 +1879,7 @@ func (s *TransactionAPI) SignTransaction(ctx context.Context, args TransactionAr if args.Nonce == nil { return nil, errors.New("nonce not specified") } - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Before actually sign the transaction, ensure the transaction fee is reasonable. @@ -1933,7 +1928,7 @@ func (s *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, g if sendArgs.Nonce == nil { return common.Hash{}, errors.New("missing transaction nonce in transaction spec") } - if err := sendArgs.setDefaults(ctx, s.b); err != nil { + if err := sendArgs.setDefaults(ctx, s.b, false); err != nil { return common.Hash{}, err } matchTx := sendArgs.toTransaction() diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index d221c14db517..a5bf863d1d71 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -96,7 +96,7 @@ func (args *TransactionArgs) data() []byte { } // setDefaults fills in default values for unspecified tx fields. -func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { +func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, skipGasEstimation bool) error { if err := args.setBlobTxSidecar(ctx, b); err != nil { return err } @@ -136,30 +136,35 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { } } - // Estimate the gas usage if necessary. if args.Gas == nil { - // These fields are immutable during the estimation, safe to - // pass the pointer directly. - data := args.data() - callArgs := TransactionArgs{ - From: args.From, - To: args.To, - GasPrice: args.GasPrice, - MaxFeePerGas: args.MaxFeePerGas, - MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, - Value: args.Value, - Data: (*hexutil.Bytes)(&data), - AccessList: args.AccessList, - BlobFeeCap: args.BlobFeeCap, - BlobHashes: args.BlobHashes, - } - latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) - estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, b.RPCGasCap()) - if err != nil { - return err + if skipGasEstimation { // Skip gas usage estimation if a precise gas limit is not critical, e.g., in non-transaction calls. + gas := hexutil.Uint64(b.RPCGasCap()) + if gas == 0 { + gas = hexutil.Uint64(math.MaxUint64 / 2) + } + args.Gas = &gas + } else { // Estimate the gas usage otherwise. + // These fields are immutable during the estimation, safe to + // pass the pointer directly. + data := args.data() + callArgs := TransactionArgs{ + From: args.From, + To: args.To, + GasPrice: args.GasPrice, + MaxFeePerGas: args.MaxFeePerGas, + MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, + Value: args.Value, + Data: (*hexutil.Bytes)(&data), + AccessList: args.AccessList, + } + latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, b.RPCGasCap()) + if err != nil { + return err + } + args.Gas = &estimated + log.Trace("Estimate gas usage automatically", "gas", args.Gas) } - args.Gas = &estimated - log.Trace("Estimate gas usage automatically", "gas", args.Gas) } // If chain id is provided, ensure it matches the local chain id. Otherwise, set the local From 93c541ad563124e81d125c7ebe78938175229b2e Mon Sep 17 00:00:00 2001 From: Haotian <51777534+tmelhao@users.noreply.github.com> Date: Fri, 23 Feb 2024 16:57:47 +0800 Subject: [PATCH 14/91] eth/catalyst: fix wrong error message of payloadV2 after cancun (#29049) * eth/catalyst: the same error format Signed-off-by: tmelhao * eth/catalyst: wrong error message for payloadV2 post-cancun Signed-off-by: tmelhao * eth/catalyst: parentBeaconBlockRoot -> parentBlockBeaconRoot Signed-off-by: tmelhao * apply commit review Signed-off-by: tmelhao --------- Signed-off-by: tmelhao Co-authored-by: tmelhao --- eth/catalyst/api.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 44518612e83f..d16d37d3284c 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -488,7 +488,7 @@ func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.Payl // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { if api.eth.BlockChain().Config().IsCancun(api.eth.BlockChain().Config().LondonBlock, params.Timestamp) { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use new payload v2 post-shanghai")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use newPayloadV2 post-cancun")) } if api.eth.BlockChain().Config().LatestFork(params.Timestamp) == forks.Shanghai { if params.Withdrawals == nil { @@ -503,7 +503,7 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil excessBlobGas pre-cancun")) } if params.BlobGasUsed != nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil params.BlobGasUsed pre-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil blobGasUsed pre-cancun")) } return api.newPayload(params, nil, nil) } @@ -517,14 +517,14 @@ func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHas return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil excessBlobGas post-cancun")) } if params.BlobGasUsed == nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil params.BlobGasUsed post-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil blobGasUsed post-cancun")) } if versionedHashes == nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil versionedHashes post-cancun")) } if beaconRoot == nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil parentBeaconBlockRoot post-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil beaconRoot post-cancun")) } if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun { From 32d4d6e6160432be1cb9780a43253deda7708ced Mon Sep 17 00:00:00 2001 From: Roberto Bayardo Date: Mon, 26 Feb 2024 01:06:52 -0800 Subject: [PATCH 15/91] core/txpool: reject blob txs with blob fee cap below the minimum (#29081) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * make blobpool reject blob transactions with fee below the minimum * core/txpool: some minot nitpick polishes and unified error formats * core/txpool: do less big.Int constructions with the min blob cap --------- Co-authored-by: Péter Szilágyi --- core/txpool/blobpool/blobpool.go | 2 +- core/txpool/blobpool/blobpool_test.go | 18 ++++++++++++++++++ core/txpool/validation.go | 19 ++++++++++++++----- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 276c2886e2dc..3ed698c1b18f 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -402,7 +402,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres } var ( basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head)) - blobfee = uint256.MustFromBig(big.NewInt(params.BlobTxMinBlobGasprice)) + blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) ) if p.head.ExcessBlobGas != nil { blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(*p.head.ExcessBlobGas)) diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index be5833011a07..f7644c1d0ab6 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -1228,6 +1228,24 @@ func TestAdd(t *testing.T) { }, }, }, + // Blob transactions that don't meet the min blob gas price should be rejected + { + seeds: map[string]seed{ + "alice": {balance: 10000000}, + }, + adds: []addtx{ + { // New account, no previous txs, nonce 0, but blob fee cap too low + from: "alice", + tx: makeUnsignedTx(0, 1, 1, 0), + err: txpool.ErrUnderpriced, + }, + { // Same as above but blob fee cap equals minimum, should be accepted + from: "alice", + tx: makeUnsignedTx(0, 1, 1, params.BlobTxMinBlobGasprice), + err: nil, + }, + }, + }, } for i, tt := range tests { // Create a temporary folder for the persistent backend diff --git a/core/txpool/validation.go b/core/txpool/validation.go index a9bd14020bc9..8913859e846e 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -30,6 +30,12 @@ import ( "github.com/ethereum/go-ethereum/params" ) +var ( + // blobTxMinBlobGasPrice is the big.Int version of the configured protocol + // parameter to avoid constucting a new big integer for every transaction. + blobTxMinBlobGasPrice = big.NewInt(params.BlobTxMinBlobGasprice) +) + // ValidationOptions define certain differences between transaction validation // across the different pools without having to duplicate those checks. type ValidationOptions struct { @@ -101,15 +107,17 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return err } if tx.Gas() < intrGas { - return fmt.Errorf("%w: needed %v, allowed %v", core.ErrIntrinsicGas, intrGas, tx.Gas()) + return fmt.Errorf("%w: gas %v, minimum needed %v", core.ErrIntrinsicGas, tx.Gas(), intrGas) } - // Ensure the gasprice is high enough to cover the requirement of the calling - // pool and/or block producer + // Ensure the gasprice is high enough to cover the requirement of the calling pool if tx.GasTipCapIntCmp(opts.MinTip) < 0 { - return fmt.Errorf("%w: tip needed %v, tip permitted %v", ErrUnderpriced, opts.MinTip, tx.GasTipCap()) + return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrUnderpriced, tx.GasTipCap(), opts.MinTip) } - // Ensure blob transactions have valid commitments if tx.Type() == types.BlobTxType { + // Ensure the blob fee cap satisfies the minimum blob gas price + if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 { + return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrUnderpriced, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) + } sidecar := tx.BlobTxSidecar() if sidecar == nil { return fmt.Errorf("missing sidecar in blob transaction") @@ -123,6 +131,7 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types if len(hashes) > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob { return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) } + // Ensure commitments, proofs and hashes are valid if err := validateBlobSidecar(hashes, sidecar); err != nil { return err } From 26724fc2aaf0cf8711c25ca664c0451f68d977fe Mon Sep 17 00:00:00 2001 From: Qt Date: Mon, 26 Feb 2024 17:25:35 +0800 Subject: [PATCH 16/91] p2p, log, rpc: use errors.New to replace fmt.Errorf with no parameters (#29074) --- log/logger_test.go | 5 +++-- p2p/server.go | 4 ++-- p2p/transport.go | 3 ++- rpc/types.go | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/log/logger_test.go b/log/logger_test.go index a633f5ad7a4c..ff981fd018ca 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -2,6 +2,7 @@ package log import ( "bytes" + "errors" "fmt" "io" "math/big" @@ -77,7 +78,7 @@ func benchmarkLogger(b *testing.B, l Logger) { tt = time.Now() bigint = big.NewInt(100) nilbig *big.Int - err = fmt.Errorf("Oh nooes it's crap") + err = errors.New("Oh nooes it's crap") ) b.ReportAllocs() b.ResetTimer() @@ -106,7 +107,7 @@ func TestLoggerOutput(t *testing.T) { tt = time.Time{} bigint = big.NewInt(100) nilbig *big.Int - err = fmt.Errorf("Oh nooes it's crap") + err = errors.New("Oh nooes it's crap") smallUint = uint256.NewInt(500_000) bigUint = &uint256.Int{0xff, 0xff, 0xff, 0xff} ) diff --git a/p2p/server.go b/p2p/server.go index 8f42765a8c26..975a3bb91662 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -914,13 +914,13 @@ func (srv *Server) checkInboundConn(remoteIP net.IP) error { } // Reject connections that do not match NetRestrict. if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { - return fmt.Errorf("not in netrestrict list") + return errors.New("not in netrestrict list") } // Reject Internet peers that try too often. now := srv.clock.Now() srv.inboundHistory.expire(now, nil) if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { - return fmt.Errorf("too many attempts") + return errors.New("too many attempts") } srv.inboundHistory.add(remoteIP.String(), now.Add(inboundThrottleTime)) return nil diff --git a/p2p/transport.go b/p2p/transport.go index 4f6bb569bfd3..5fc7686feb06 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -19,6 +19,7 @@ package p2p import ( "bytes" "crypto/ecdsa" + "errors" "fmt" "io" "net" @@ -157,7 +158,7 @@ func readProtocolHandshake(rw MsgReader) (*protoHandshake, error) { return nil, err } if msg.Size > baseProtocolMaxMsgSize { - return nil, fmt.Errorf("message too big") + return nil, errors.New("message too big") } if msg.Code == discMsg { // Disconnect before protocol handshake is valid according to the diff --git a/rpc/types.go b/rpc/types.go index f88c37c59dad..d12408178615 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -19,6 +19,7 @@ package rpc import ( "context" "encoding/json" + "errors" "fmt" "math" "strings" @@ -104,7 +105,7 @@ func (bn *BlockNumber) UnmarshalJSON(data []byte) error { return err } if blckNum > math.MaxInt64 { - return fmt.Errorf("block number larger than int64") + return errors.New("block number larger than int64") } *bn = BlockNumber(blckNum) return nil @@ -154,7 +155,7 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { err := json.Unmarshal(data, &e) if err == nil { if e.BlockNumber != nil && e.BlockHash != nil { - return fmt.Errorf("cannot specify both BlockHash and BlockNumber, choose one or the other") + return errors.New("cannot specify both BlockHash and BlockNumber, choose one or the other") } bnh.BlockNumber = e.BlockNumber bnh.BlockHash = e.BlockHash @@ -202,7 +203,7 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { return err } if blckNum > math.MaxInt64 { - return fmt.Errorf("blocknumber too high") + return errors.New("blocknumber too high") } bn := BlockNumber(blckNum) bnh.BlockNumber = &bn From edffacca8f97d23298636e225d477818e58eafe7 Mon Sep 17 00:00:00 2001 From: cui <523516579@qq.com> Date: Mon, 26 Feb 2024 17:59:03 +0800 Subject: [PATCH 17/91] =?UTF-8?q?eth/catalyst:=20enable=20some=20commented?= =?UTF-8?q?-out=20testcases=C2=A0=C2=A0=20(#29073)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eth/catalyst/api_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 9856118eae3f..cc1258ca55bf 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -262,11 +262,8 @@ func TestInvalidPayloadTimestamp(t *testing.T) { {0, true}, {parent.Time, true}, {parent.Time - 1, true}, - - // TODO (MariusVanDerWijden) following tests are currently broken, - // fixed in upcoming merge-kiln-v2 pr - //{parent.Time() + 1, false}, - //{uint64(time.Now().Unix()) + uint64(time.Minute), false}, + {parent.Time + 1, false}, + {uint64(time.Now().Unix()) + uint64(time.Minute), false}, } for i, test := range tests { From 8bca93e82c59d04f23b0237292d17fe728f20a5b Mon Sep 17 00:00:00 2001 From: maskpp Date: Mon, 26 Feb 2024 18:02:18 +0800 Subject: [PATCH 18/91] internal/ethapi: pass blob hashes to gas estimation (#29085) --- internal/ethapi/transaction_args.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index a5bf863d1d71..bae1c6864159 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -156,6 +156,8 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, skipGas Value: args.Value, Data: (*hexutil.Bytes)(&data), AccessList: args.AccessList, + BlobFeeCap: args.BlobFeeCap, + BlobHashes: args.BlobHashes, } latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, b.RPCGasCap()) From 821d70240d191ff451a813287a377466337a3cee Mon Sep 17 00:00:00 2001 From: Justin Dhillon Date: Mon, 26 Feb 2024 02:03:59 -0800 Subject: [PATCH 19/91] cmd/clef: add spaces in README.md table (#29077) Add space after links in so they are clickable in vscode. --- cmd/clef/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/clef/README.md b/cmd/clef/README.md index 3a43db8c95a3..cf0926513603 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -916,7 +916,7 @@ There are a couple of implementation for a UI. We'll try to keep this list up to | Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters| | ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- | -| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| -| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | -| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | -| Clef UI| https://github.com/ethereum/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| +| QtSigner| https://github.com/holiman/qtsigner/ | Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| +| GtkSigner| https://github.com/holiman/gtksigner | Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | +| Frame | https://github.com/floating/frame/commits/go-signer | Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | +| Clef UI| https://github.com/ethereum/clef-ui | Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| From c1f59b98f6b0351339767d71953eb4eb5d19c496 Mon Sep 17 00:00:00 2001 From: cui <523516579@qq.com> Date: Mon, 26 Feb 2024 20:22:13 +0800 Subject: [PATCH 20/91] eth/catalyst: remove variable in tx conversion loop (#29076) --- eth/catalyst/api.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index d16d37d3284c..58566a47fc6c 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -879,8 +879,7 @@ func getBody(block *types.Block) *engine.ExecutionPayloadBodyV1 { ) for j, tx := range body.Transactions { - data, _ := tx.MarshalBinary() - txs[j] = hexutil.Bytes(data) + txs[j], _ = tx.MarshalBinary() } // Post-shanghai withdrawals MUST be set to empty slice instead of nil From 63aaac81007ad46b208570c17cae78b7f60931d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 26 Feb 2024 14:27:56 +0200 Subject: [PATCH 21/91] core/txpool/blobpool: reduce default database cap for rollout (#29090) xcore/txpool/blobpool: reduce default database cap for rollout --- core/txpool/blobpool/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/txpool/blobpool/config.go b/core/txpool/blobpool/config.go index 99a2002a303f..1d180739cdfb 100644 --- a/core/txpool/blobpool/config.go +++ b/core/txpool/blobpool/config.go @@ -30,8 +30,8 @@ type Config struct { // DefaultConfig contains the default configurations for the transaction pool. var DefaultConfig = Config{ Datadir: "blobpool", - Datacap: 10 * 1024 * 1024 * 1024, - PriceBump: 100, // either have patience or be aggressive, no mushy ground + Datacap: 10 * 1024 * 1024 * 1024 / 4, // TODO(karalabe): /4 handicap for rollout, gradually bump back up to 10GB + PriceBump: 100, // either have patience or be aggressive, no mushy ground } // sanitize checks the provided user configurations and changes anything that's From 45a272c7b96cb260528bbc2e31d657488f97c4b0 Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 27 Feb 2024 00:34:45 +0800 Subject: [PATCH 22/91] core/txpool: no need to log loud rotate if no local txs (#29083) * core/txpool: no need to run rotate if no local txs Signed-off-by: jsvisa * Revert "core/txpool: no need to run rotate if no local txs" This reverts commit 17fab173883168c586d57ca9c05dfcbd9e7831b4. Signed-off-by: jsvisa * use Debug if todo is empty Signed-off-by: jsvisa --------- Signed-off-by: jsvisa --- core/txpool/legacypool/journal.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/txpool/legacypool/journal.go b/core/txpool/legacypool/journal.go index f04ab8fc14e5..899ed00bcced 100644 --- a/core/txpool/legacypool/journal.go +++ b/core/txpool/legacypool/journal.go @@ -164,7 +164,12 @@ func (journal *journal) rotate(all map[common.Address]types.Transactions) error return err } journal.writer = sink - log.Info("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all)) + + logger := log.Info + if len(all) == 0 { + logger = log.Debug + } + logger("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all)) return nil } From 5a0f468f8cb15b939bd85445d33c614a36942a8e Mon Sep 17 00:00:00 2001 From: Andrei Silviu Dragnea Date: Tue, 27 Feb 2024 10:29:12 +0100 Subject: [PATCH 23/91] eth/tracers: Fix callTracer logs on onlyTopCall == true (#29068) --- eth/tracers/native/call.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index f85cf6206a72..be9b58a4cd3c 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -161,7 +161,7 @@ func (t *callTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, sco return } // Avoid processing nested calls when only caring about top call - if t.config.OnlyTopCall && depth > 0 { + if t.config.OnlyTopCall && depth > 1 { return } // Skip if tracing was interrupted From 51b479e56459d663a12f95fd8eaba82716c0d5ce Mon Sep 17 00:00:00 2001 From: Roberto Bayardo Date: Tue, 27 Feb 2024 03:27:50 -0800 Subject: [PATCH 24/91] core/txpool: elevate the 'already reserved' error into a constant (#29095) declare the 'already reserved' error in errors.go --- core/txpool/errors.go | 6 ++++++ core/txpool/txpool.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/txpool/errors.go b/core/txpool/errors.go index 61daa999ffdc..3a6a913976ca 100644 --- a/core/txpool/errors.go +++ b/core/txpool/errors.go @@ -54,4 +54,10 @@ var ( // ErrFutureReplacePending is returned if a future transaction replaces a pending // one. Future transactions should only be able to replace other future transactions. ErrFutureReplacePending = errors.New("future transaction tries to replace pending") + + // ErrAlreadyReserved is returned if the sender address has a pending transaction + // in a different subpool. For example, this error is returned in response to any + // input transaction of non-blob type when a blob transaction from this sender + // remains pending (and vice-versa). + ErrAlreadyReserved = errors.New("address already reserved") ) diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 8bf3e0a51261..be7435247d92 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -122,7 +122,7 @@ func (p *TxPool) reserver(id int, subpool SubPool) AddressReserver { log.Error("pool attempted to reserve already-owned address", "address", addr) return nil // Ignore fault to give the pool a chance to recover while the bug gets fixed } - return errors.New("address already reserved") + return ErrAlreadyReserved } p.reservations[addr] = subpool if metrics.Enabled { From 9038ba69428a6ecada1f2acace6981854482748b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 27 Feb 2024 13:50:30 +0200 Subject: [PATCH 25/91] params: release Geth v1.13.14 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 34ba3f74200b..09368cd9faac 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 14 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 13 // Minor version component of the current release + VersionPatch = 14 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 57d2b552c74dbd03b9909e6b8cd7b3de1f8b40e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 27 Feb 2024 13:53:30 +0200 Subject: [PATCH 26/91] params: begin v1.13.15 cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 09368cd9faac..671037a82b7e 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 14 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 13 // Minor version component of the current release + VersionPatch = 15 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 02d77c98f9e1efaf3fede313b0e9183dc54562b6 Mon Sep 17 00:00:00 2001 From: cui <523516579@qq.com> Date: Wed, 28 Feb 2024 15:25:12 +0800 Subject: [PATCH 27/91] core: using math.MaxUint64 instead of 0xffffffffffffffff (#29094) --- core/vm/instructions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index b8055de6bce7..ac3ea4bcd62b 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -305,7 +305,7 @@ func opCallDataCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext ) dataOffset64, overflow := dataOffset.Uint64WithOverflow() if overflow { - dataOffset64 = 0xffffffffffffffff + dataOffset64 = math.MaxUint64 } // These values are checked for overflow during gas cost calculation memOffset64 := memOffset.Uint64() From 170fcd80c6f5d07d7d839e895765de193c34a8b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Feb 2024 10:01:52 +0200 Subject: [PATCH 28/91] params: being major version bump cycle --- params/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/params/version.go b/params/version.go index 671037a82b7e..a49385da7d56 100644 --- a/params/version.go +++ b/params/version.go @@ -22,8 +22,8 @@ import ( const ( VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 15 // Patch version component of the current release + VersionMinor = 14 // Minor version component of the current release + VersionPatch = 0 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) From 49623bd4697f5b333ae977968186d0717f918927 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 28 Feb 2024 20:23:52 +0800 Subject: [PATCH 29/91] core, triedb/pathdb: calculate the size for batch pre-allocation (#29106) * core, triedb/pathdb: calculate the size for batch pre-allocation * triedb/pathdb: address comment --- core/rawdb/schema.go | 30 +++++++++++++++--------------- triedb/pathdb/nodebuffer.go | 15 ++++++++++++++- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 11cf5b40fef6..dbf010be0ca8 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -113,8 +113,8 @@ var ( skeletonHeaderPrefix = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header // Path-based storage scheme of merkle patricia trie. - trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node - trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node + TrieNodeAccountPrefix = []byte("A") // TrieNodeAccountPrefix + hexPath -> trie node + TrieNodeStoragePrefix = []byte("O") // TrieNodeStoragePrefix + accountHash + hexPath -> trie node stateIDPrefix = []byte("L") // stateIDPrefix + state root -> state id PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage @@ -265,15 +265,15 @@ func stateIDKey(root common.Hash) []byte { return append(stateIDPrefix, root.Bytes()...) } -// accountTrieNodeKey = trieNodeAccountPrefix + nodePath. +// accountTrieNodeKey = TrieNodeAccountPrefix + nodePath. func accountTrieNodeKey(path []byte) []byte { - return append(trieNodeAccountPrefix, path...) + return append(TrieNodeAccountPrefix, path...) } -// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath. +// storageTrieNodeKey = TrieNodeStoragePrefix + accountHash + nodePath. func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { - buf := make([]byte, len(trieNodeStoragePrefix)+common.HashLength+len(path)) - n := copy(buf, trieNodeStoragePrefix) + buf := make([]byte, len(TrieNodeStoragePrefix)+common.HashLength+len(path)) + n := copy(buf, TrieNodeStoragePrefix) n += copy(buf[n:], accountHash.Bytes()) copy(buf[n:], path) return buf @@ -294,16 +294,16 @@ func IsLegacyTrieNode(key []byte, val []byte) bool { // account trie node in path-based state scheme, and returns the resolved // node path if so. func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) { - if !bytes.HasPrefix(key, trieNodeAccountPrefix) { + if !bytes.HasPrefix(key, TrieNodeAccountPrefix) { return false, nil } // The remaining key should only consist a hex node path // whose length is in the range 0 to 64 (64 is excluded // since leaves are always wrapped with shortNode). - if len(key) >= len(trieNodeAccountPrefix)+common.HashLength*2 { + if len(key) >= len(TrieNodeAccountPrefix)+common.HashLength*2 { return false, nil } - return true, key[len(trieNodeAccountPrefix):] + return true, key[len(TrieNodeAccountPrefix):] } // IsAccountTrieNode reports whether a provided database entry is an account @@ -317,20 +317,20 @@ func IsAccountTrieNode(key []byte) bool { // trie node in path-based state scheme, and returns the resolved account hash // and node path if so. func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) { - if !bytes.HasPrefix(key, trieNodeStoragePrefix) { + if !bytes.HasPrefix(key, TrieNodeStoragePrefix) { return false, common.Hash{}, nil } // The remaining key consists of 2 parts: // - 32 bytes account hash // - hex node path whose length is in the range 0 to 64 - if len(key) < len(trieNodeStoragePrefix)+common.HashLength { + if len(key) < len(TrieNodeStoragePrefix)+common.HashLength { return false, common.Hash{}, nil } - if len(key) >= len(trieNodeStoragePrefix)+common.HashLength+common.HashLength*2 { + if len(key) >= len(TrieNodeStoragePrefix)+common.HashLength+common.HashLength*2 { return false, common.Hash{}, nil } - accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength]) - return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:] + accountHash := common.BytesToHash(key[len(TrieNodeStoragePrefix) : len(TrieNodeStoragePrefix)+common.HashLength]) + return true, accountHash, key[len(TrieNodeStoragePrefix)+common.HashLength:] } // IsStorageTrieNode reports whether a provided database entry is a storage diff --git a/triedb/pathdb/nodebuffer.go b/triedb/pathdb/nodebuffer.go index 4a7d328b9afb..8f84c2b44207 100644 --- a/triedb/pathdb/nodebuffer.go +++ b/triedb/pathdb/nodebuffer.go @@ -204,6 +204,19 @@ func (b *nodebuffer) setSize(size int, db ethdb.KeyValueStore, clean *fastcache. return b.flush(db, clean, id, false) } +// allocBatch returns a database batch with pre-allocated buffer. +func (b *nodebuffer) allocBatch(db ethdb.KeyValueStore) ethdb.Batch { + var metasize int + for owner, nodes := range b.nodes { + if owner == (common.Hash{}) { + metasize += len(nodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix + } else { + metasize += len(nodes) * (len(rawdb.TrieNodeStoragePrefix) + common.HashLength) // database key prefix + owner + } + } + return db.NewBatchWithSize((metasize + int(b.size)) * 11 / 10) // extra 10% for potential pebble internal stuff +} + // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error { @@ -217,7 +230,7 @@ func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id ui } var ( start = time.Now() - batch = db.NewBatchWithSize(int(b.size)) + batch = b.allocBatch(db) ) nodes := writeNodes(batch, b.nodes, clean) rawdb.WritePersistentStateID(batch, id) From 5bae14f9df498243091078fc8d3ea6ab99669087 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 28 Feb 2024 20:40:28 +0800 Subject: [PATCH 30/91] triedb/pathdb: fix panic in recoverable (#29107) * triedb/pathdb: fix panic in recoverable * triedb/pathdb: add todo * triedb/pathdb: rename * triedb/pathdb: rename --- triedb/pathdb/database.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index f2d6cea635a9..307f307df53d 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -391,17 +391,23 @@ func (db *Database) Recoverable(root common.Hash) bool { if *id >= dl.stateID() { return false } + // This is a temporary workaround for the unavailability of the freezer in + // dev mode. As a consequence, the Pathdb loses the ability for deep reorg + // in certain cases. + // TODO(rjl493456442): Implement the in-memory ancient store. + if db.freezer == nil { + return false + } // Ensure the requested state is a canonical state and all state // histories in range [id+1, disklayer.ID] are present and complete. - parent := root return checkHistories(db.freezer, *id+1, dl.stateID()-*id, func(m *meta) error { - if m.parent != parent { + if m.parent != root { return errors.New("unexpected state history") } if len(m.incomplete) > 0 { return errors.New("incomplete state history") } - parent = m.root + root = m.root return nil }) == nil } From 9986a69c25452ff0e7ce323446b215e2d0075185 Mon Sep 17 00:00:00 2001 From: buddho Date: Thu, 29 Feb 2024 01:38:21 +0800 Subject: [PATCH 31/91] internal/ethapi: pass in accesslist in test (#29089) Co-authored-by: Sina Mahmoodi --- internal/ethapi/api_test.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index a6f7405eb363..8ffa638a6b5c 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -1272,10 +1272,14 @@ func TestFillBlobTransaction(t *testing.T) { func argsFromTransaction(tx *types.Transaction, from common.Address) TransactionArgs { var ( - gas = tx.Gas() - nonce = tx.Nonce() - input = tx.Data() + gas = tx.Gas() + nonce = tx.Nonce() + input = tx.Data() + accessList *types.AccessList ) + if acl := tx.AccessList(); acl != nil { + accessList = &acl + } return TransactionArgs{ From: &from, To: tx.To(), @@ -1286,10 +1290,9 @@ func argsFromTransaction(tx *types.Transaction, from common.Address) Transaction Nonce: (*hexutil.Uint64)(&nonce), Input: (*hexutil.Bytes)(&input), ChainID: (*hexutil.Big)(tx.ChainId()), - // TODO: impl accessList conversion - //AccessList: tx.AccessList(), - BlobFeeCap: (*hexutil.Big)(tx.BlobGasFeeCap()), - BlobHashes: tx.BlobHashes(), + AccessList: accessList, + BlobFeeCap: (*hexutil.Big)(tx.BlobGasFeeCap()), + BlobHashes: tx.BlobHashes(), } } From 1883438964a7a4c68cee1de619526e8bc1e68b30 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Wed, 28 Feb 2024 11:59:16 -0700 Subject: [PATCH 32/91] eth/catalyst: return invalid payload attributes instead of invalid parms for bad fcu payload (#29115) --- eth/catalyst/api.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 58566a47fc6c..fea9d34cb83e 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -190,21 +190,21 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, pa // attributes. It supports both PayloadAttributesV1 and PayloadAttributesV2. func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { + if params.BeaconRoot != nil { + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("unexpected beacon root")) + } switch api.eth.BlockChain().Config().LatestFork(params.Timestamp) { case forks.Paris: if params.Withdrawals != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals before shanghai")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("withdrawals before shanghai")) } case forks.Shanghai: if params.Withdrawals == nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing withdrawals")) } default: return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV2 must only be called with paris and shanghai payloads")) } - if params.BeaconRoot != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("unexpected beacon root")) - } } return api.forkchoiceUpdated(update, params, engine.PayloadV2, false) } @@ -213,15 +213,11 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, pa // in the payload attributes. It supports only PayloadAttributesV3. func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { - // TODO(matt): according to https://github.com/ethereum/execution-apis/pull/498, - // payload attributes that are invalid should return error - // engine.InvalidPayloadAttributes. Once hive updates this, we should update - // on our end. if params.Withdrawals == nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing withdrawals")) } if params.BeaconRoot == nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing beacon root")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing beacon root")) } if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun { return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV3 must only be called for cancun payloads")) From dbc27a199f411fc620eeb8589fd75a144f83ee8c Mon Sep 17 00:00:00 2001 From: cui fliter Date: Thu, 29 Feb 2024 17:29:06 +0800 Subject: [PATCH 33/91] all: fix function names in docs (#29128) Signed-off-by: cui fliter --- eth/peerset.go | 2 +- eth/protocols/eth/dispatcher.go | 2 +- internal/era/iterator.go | 2 +- metrics/sample.go | 2 +- p2p/enode/nodedb.go | 4 ++-- rpc/handler.go | 2 +- signer/core/signed_data.go | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/eth/peerset.go b/eth/peerset.go index c0c11e3e85ee..c56a7223e964 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -100,7 +100,7 @@ func (ps *peerSet) registerSnapExtension(peer *snap.Peer) error { return nil } -// waitExtensions blocks until all satellite protocols are connected and tracked +// waitSnapExtension blocks until all satellite protocols are connected and tracked // by the peerset. func (ps *peerSet) waitSnapExtension(peer *eth.Peer) (*snap.Peer, error) { // If the peer does not support a compatible `snap`, don't wait diff --git a/eth/protocols/eth/dispatcher.go b/eth/protocols/eth/dispatcher.go index ae98820cd6ac..146eec3f6085 100644 --- a/eth/protocols/eth/dispatcher.go +++ b/eth/protocols/eth/dispatcher.go @@ -136,7 +136,7 @@ func (p *Peer) dispatchRequest(req *Request) error { } } -// dispatchRequest fulfils a pending request and delivers it to the requested +// dispatchResponse fulfils a pending request and delivers it to the requested // sink. func (p *Peer) dispatchResponse(res *Response, metadata func() interface{}) error { resOp := &response{ diff --git a/internal/era/iterator.go b/internal/era/iterator.go index e74a8154b1a6..5dfc12445f79 100644 --- a/internal/era/iterator.go +++ b/internal/era/iterator.go @@ -30,7 +30,7 @@ type Iterator struct { inner *RawIterator } -// NewRawIterator returns a new Iterator instance. Next must be immediately +// NewIterator returns a new Iterator instance. Next must be immediately // called on new iterators to load the first item. func NewIterator(e *Era) (*Iterator, error) { inner, err := NewRawIterator(e) diff --git a/metrics/sample.go b/metrics/sample.go index 5398dd42d5de..bb81e105cf9e 100644 --- a/metrics/sample.go +++ b/metrics/sample.go @@ -148,7 +148,7 @@ func (NilSample) Clear() {} func (NilSample) Snapshot() SampleSnapshot { return (*emptySnapshot)(nil) } func (NilSample) Update(v int64) {} -// SamplePercentiles returns an arbitrary percentile of the slice of int64. +// SamplePercentile returns an arbitrary percentile of the slice of int64. func SamplePercentile(values []int64, p float64) float64 { return CalculatePercentiles(values, []float64{p})[0] } diff --git a/p2p/enode/nodedb.go b/p2p/enode/nodedb.go index 7e7fb69b293a..6d55ce17f130 100644 --- a/p2p/enode/nodedb.go +++ b/p2p/enode/nodedb.go @@ -84,7 +84,7 @@ func OpenDB(path string) (*DB, error) { return newPersistentDB(path) } -// newMemoryNodeDB creates a new in-memory node database without a persistent backend. +// newMemoryDB creates a new in-memory node database without a persistent backend. func newMemoryDB() (*DB, error) { db, err := leveldb.Open(storage.NewMemStorage(), nil) if err != nil { @@ -93,7 +93,7 @@ func newMemoryDB() (*DB, error) { return &DB{lvl: db, quit: make(chan struct{})}, nil } -// newPersistentNodeDB creates/opens a leveldb backed persistent node database, +// newPersistentDB creates/opens a leveldb backed persistent node database, // also flushing its contents in case of a version mismatch. func newPersistentDB(path string) (*DB, error) { opts := &opt.Options{OpenFilesCacheCapacity: 5} diff --git a/rpc/handler.go b/rpc/handler.go index f44e4d7b01d8..792581cbc0ad 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -324,7 +324,7 @@ func (h *handler) addRequestOp(op *requestOp) { } } -// removeRequestOps stops waiting for the given request IDs. +// removeRequestOp stops waiting for the given request IDs. func (h *handler) removeRequestOp(op *requestOp) { for _, id := range op.ids { delete(h.respWait, string(id)) diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index c6ae7b12743f..f8b3c9d86d1d 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -260,7 +260,7 @@ func fromHex(data any) ([]byte, error) { return nil, fmt.Errorf("wrong type %T", data) } -// typeDataRequest tries to convert the data into a SignDataRequest. +// typedDataRequest tries to convert the data into a SignDataRequest. func typedDataRequest(data any) (*SignDataRequest, error) { var typedData apitypes.TypedData if td, ok := data.(apitypes.TypedData); ok { From 28d55218f7d793c184f4220a16a60e309caa70af Mon Sep 17 00:00:00 2001 From: Ng Wei Han <47109095+weiihann@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:56:17 +0800 Subject: [PATCH 34/91] cmd/geth: parseDumpConfig should not return closed db (#29100) * cmd: parseDumpConfig should not return closed db * fix lint --- cmd/geth/chaincmd.go | 24 ++++++++++++------------ cmd/geth/snapshot.go | 5 ++++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index d333c175599d..c8041d563a1a 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -514,13 +514,10 @@ func importPreimages(ctx *cli.Context) error { return nil } -func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) { - db := utils.MakeChainDatabase(ctx, stack, true) - defer db.Close() - +func parseDumpConfig(ctx *cli.Context, stack *node.Node, db ethdb.Database) (*state.DumpConfig, common.Hash, error) { var header *types.Header if ctx.NArg() > 1 { - return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) + return nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) } if ctx.NArg() == 1 { arg := ctx.Args().First() @@ -529,17 +526,17 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth if number := rawdb.ReadHeaderNumber(db, hash); number != nil { header = rawdb.ReadHeader(db, hash, *number) } else { - return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash) + return nil, common.Hash{}, fmt.Errorf("block %x not found", hash) } } else { number, err := strconv.ParseUint(arg, 10, 64) if err != nil { - return nil, nil, common.Hash{}, err + return nil, common.Hash{}, err } if hash := rawdb.ReadCanonicalHash(db, number); hash != (common.Hash{}) { header = rawdb.ReadHeader(db, hash, number) } else { - return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) + return nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) } } } else { @@ -547,7 +544,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth header = rawdb.ReadHeadHeader(db) } if header == nil { - return nil, nil, common.Hash{}, errors.New("no head block found") + return nil, common.Hash{}, errors.New("no head block found") } startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name)) var start common.Hash @@ -559,7 +556,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth start = crypto.Keccak256Hash(startArg) log.Info("Converting start-address to hash", "address", common.BytesToAddress(startArg), "hash", start.Hex()) default: - return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) + return nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) } var conf = &state.DumpConfig{ SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name), @@ -571,14 +568,17 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(), "skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage, "start", hexutil.Encode(conf.Start), "limit", conf.Max) - return conf, db, header.Root, nil + return conf, header.Root, nil } func dump(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - conf, db, root, err := parseDumpConfig(ctx, stack) + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + conf, root, err := parseDumpConfig(ctx, stack, db) if err != nil { return err } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 4284005a0221..1e0933e46f59 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -541,7 +541,10 @@ func dumpState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - conf, db, root, err := parseDumpConfig(ctx, stack) + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + conf, root, err := parseDumpConfig(ctx, stack, db) if err != nil { return err } From db4cf6916606e07d908af44e405257925dd9265e Mon Sep 17 00:00:00 2001 From: yzb <335357057@qq.com> Date: Thu, 29 Feb 2024 17:56:46 +0800 Subject: [PATCH 35/91] all: replace fmt.Errorf() with errors.New() if no param required (#29126) replace-fmt-errorf Co-authored-by: yzb@example.cn --- cmd/era/main.go | 7 ++++--- cmd/geth/chaincmd.go | 2 +- cmd/utils/cmd.go | 2 +- core/txpool/validation.go | 5 +++-- internal/era/accumulator.go | 3 ++- internal/era/builder.go | 3 ++- internal/era/e2store/e2store.go | 3 ++- internal/era/e2store/e2store_test.go | 4 ++-- internal/era/era.go | 3 ++- internal/era/iterator.go | 3 ++- miner/worker.go | 2 +- node/rpcstack.go | 5 +++-- p2p/discover/v4_udp.go | 2 +- p2p/discover/v5_udp.go | 2 +- p2p/discover/v5wire/encoding.go | 6 +++--- p2p/dnsdisc/client.go | 2 +- p2p/dnsdisc/tree.go | 3 ++- p2p/enode/idscheme.go | 4 ++-- p2p/nat/natpmp.go | 3 ++- p2p/simulations/adapters/exec.go | 2 +- signer/core/apitypes/types.go | 2 +- trie/trie_test.go | 10 +++++----- triedb/pathdb/history.go | 2 +- 23 files changed, 45 insertions(+), 35 deletions(-) diff --git a/cmd/era/main.go b/cmd/era/main.go index e27d8ccec605..c7f5de12bc1a 100644 --- a/cmd/era/main.go +++ b/cmd/era/main.go @@ -18,6 +18,7 @@ package main import ( "encoding/json" + "errors" "fmt" "math/big" "os" @@ -182,7 +183,7 @@ func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { // that the accumulator matches the expected value. func verify(ctx *cli.Context) error { if ctx.Args().Len() != 1 { - return fmt.Errorf("missing accumulators file") + return errors.New("missing accumulators file") } roots, err := readHashes(ctx.Args().First()) @@ -203,7 +204,7 @@ func verify(ctx *cli.Context) error { } if len(entries) != len(roots) { - return fmt.Errorf("number of era1 files should match the number of accumulator hashes") + return errors.New("number of era1 files should match the number of accumulator hashes") } // Verify each epoch matches the expected root. @@ -308,7 +309,7 @@ func checkAccumulator(e *era.Era) error { func readHashes(f string) ([]common.Hash, error) { b, err := os.ReadFile(f) if err != nil { - return nil, fmt.Errorf("unable to open accumulators file") + return nil, errors.New("unable to open accumulators file") } s := strings.Split(string(b), "\n") // Remove empty last element, if present. diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c8041d563a1a..17aab678768d 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -444,7 +444,7 @@ func importHistory(ctx *cli.Context) error { return fmt.Errorf("no era1 files found in %s", dir) } if len(networks) > 1 { - return fmt.Errorf("multiple networks found, use a network flag to specify desired network") + return errors.New("multiple networks found, use a network flag to specify desired network") } network = networks[0] } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 4b5716466556..37736dda8509 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -245,7 +245,7 @@ func readList(filename string) ([]string, error) { // starting from genesis. func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error { if chain.CurrentSnapBlock().Number.BitLen() != 0 { - return fmt.Errorf("history import only supported when starting from genesis") + return errors.New("history import only supported when starting from genesis") } entries, err := era.ReadDir(dir, network) if err != nil { diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 8913859e846e..63f127f55ca2 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -18,6 +18,7 @@ package txpool import ( "crypto/sha256" + "errors" "fmt" "math/big" @@ -120,13 +121,13 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types } sidecar := tx.BlobTxSidecar() if sidecar == nil { - return fmt.Errorf("missing sidecar in blob transaction") + return errors.New("missing sidecar in blob transaction") } // Ensure the number of items in the blob transaction and various side // data match up before doing any expensive validations hashes := tx.BlobHashes() if len(hashes) == 0 { - return fmt.Errorf("blobless blob transaction") + return errors.New("blobless blob transaction") } if len(hashes) > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob { return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) diff --git a/internal/era/accumulator.go b/internal/era/accumulator.go index 19e03973f1f5..2ece2755e12d 100644 --- a/internal/era/accumulator.go +++ b/internal/era/accumulator.go @@ -17,6 +17,7 @@ package era import ( + "errors" "fmt" "math/big" @@ -28,7 +29,7 @@ import ( // accumulator of header records. func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, error) { if len(hashes) != len(tds) { - return common.Hash{}, fmt.Errorf("must have equal number hashes as td values") + return common.Hash{}, errors.New("must have equal number hashes as td values") } if len(hashes) > MaxEra1Size { return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEra1Size) diff --git a/internal/era/builder.go b/internal/era/builder.go index 9217c049f33b..75782a08c251 100644 --- a/internal/era/builder.go +++ b/internal/era/builder.go @@ -18,6 +18,7 @@ package era import ( "bytes" "encoding/binary" + "errors" "fmt" "io" "math/big" @@ -158,7 +159,7 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm // corresponding e2store entries. func (b *Builder) Finalize() (common.Hash, error) { if b.startNum == nil { - return common.Hash{}, fmt.Errorf("finalize called on empty builder") + return common.Hash{}, errors.New("finalize called on empty builder") } // Compute accumulator root and write entry. root, err := ComputeAccumulator(b.hashes, b.tds) diff --git a/internal/era/e2store/e2store.go b/internal/era/e2store/e2store.go index d85b3e44e97d..8e4d5dd24a55 100644 --- a/internal/era/e2store/e2store.go +++ b/internal/era/e2store/e2store.go @@ -18,6 +18,7 @@ package e2store import ( "encoding/binary" + "errors" "fmt" "io" ) @@ -160,7 +161,7 @@ func (r *Reader) ReadMetadataAt(off int64) (typ uint16, length uint32, err error // Check reserved bytes of header. if b[6] != 0 || b[7] != 0 { - return 0, 0, fmt.Errorf("reserved bytes are non-zero") + return 0, 0, errors.New("reserved bytes are non-zero") } return typ, length, nil diff --git a/internal/era/e2store/e2store_test.go b/internal/era/e2store/e2store_test.go index febcffe4cf2c..b0803493c7cd 100644 --- a/internal/era/e2store/e2store_test.go +++ b/internal/era/e2store/e2store_test.go @@ -18,7 +18,7 @@ package e2store import ( "bytes" - "fmt" + "errors" "io" "testing" @@ -92,7 +92,7 @@ func TestDecode(t *testing.T) { }, { // basic invalid decoding have: "ffff000000000001", - err: fmt.Errorf("reserved bytes are non-zero"), + err: errors.New("reserved bytes are non-zero"), }, { // no more entries to read, returns EOF have: "", diff --git a/internal/era/era.go b/internal/era/era.go index a0e701b7e0f9..2099c2d575c7 100644 --- a/internal/era/era.go +++ b/internal/era/era.go @@ -18,6 +18,7 @@ package era import ( "encoding/binary" + "errors" "fmt" "io" "math/big" @@ -127,7 +128,7 @@ func (e *Era) Close() error { func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds") + return nil, errors.New("out-of-bounds") } off, err := e.readOffset(num) if err != nil { diff --git a/internal/era/iterator.go b/internal/era/iterator.go index 5dfc12445f79..d90e9586a4e6 100644 --- a/internal/era/iterator.go +++ b/internal/era/iterator.go @@ -17,6 +17,7 @@ package era import ( + "errors" "fmt" "io" "math/big" @@ -61,7 +62,7 @@ func (it *Iterator) Error() error { // Block returns the block for the iterator's current position. func (it *Iterator) Block() (*types.Block, error) { if it.inner.Header == nil || it.inner.Body == nil { - return nil, fmt.Errorf("header and body must be non-nil") + return nil, errors.New("header and body must be non-nil") } var ( header types.Header diff --git a/miner/worker.go b/miner/worker.go index 9a36106231e9..134f91cafc4b 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -947,7 +947,7 @@ func (w *worker) prepareWork(genParams *generateParams) (*environment, error) { if genParams.parentHash != (common.Hash{}) { block := w.chain.GetBlockByHash(genParams.parentHash) if block == nil { - return nil, fmt.Errorf("missing parent") + return nil, errors.New("missing parent") } parent = block.Header() } diff --git a/node/rpcstack.go b/node/rpcstack.go index d80d5271a7fa..253db0d564a6 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -19,6 +19,7 @@ package node import ( "compress/gzip" "context" + "errors" "fmt" "io" "net" @@ -299,7 +300,7 @@ func (h *httpServer) enableRPC(apis []rpc.API, config httpConfig) error { defer h.mu.Unlock() if h.rpcAllowed() { - return fmt.Errorf("JSON-RPC over HTTP is already enabled") + return errors.New("JSON-RPC over HTTP is already enabled") } // Create RPC server and handler. @@ -335,7 +336,7 @@ func (h *httpServer) enableWS(apis []rpc.API, config wsConfig) error { defer h.mu.Unlock() if h.wsAllowed() { - return fmt.Errorf("JSON-RPC over WebSocket is already enabled") + return errors.New("JSON-RPC over WebSocket is already enabled") } // Create RPC server and handler. srv := rpc.NewServer() diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 988f16b01df2..44b1f5305c45 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -364,7 +364,7 @@ func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { return nil, err } if respN.ID() != n.ID() { - return nil, fmt.Errorf("invalid ID in response record") + return nil, errors.New("invalid ID in response record") } if respN.Seq() < n.Seq() { return n, nil // response record is older diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 8b3e33d37cf7..71f8d8dd0899 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -442,7 +442,7 @@ func (t *UDPv5) verifyResponseNode(c *callV5, r *enr.Record, distances []uint, s } } if _, ok := seen[node.ID()]; ok { - return nil, fmt.Errorf("duplicate record") + return nil, errors.New("duplicate record") } seen[node.ID()] = struct{}{} return node, nil diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 5108910620e0..904a3ddec6f2 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -367,11 +367,11 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey // key is part of the ID nonce signature. var remotePubkey = new(ecdsa.PublicKey) if err := challenge.Node.Load((*enode.Secp256k1)(remotePubkey)); err != nil { - return nil, nil, fmt.Errorf("can't find secp256k1 key for recipient") + return nil, nil, errors.New("can't find secp256k1 key for recipient") } ephkey, err := c.sc.ephemeralKeyGen() if err != nil { - return nil, nil, fmt.Errorf("can't generate ephemeral key") + return nil, nil, errors.New("can't generate ephemeral key") } ephpubkey := EncodePubkey(&ephkey.PublicKey) auth.pubkey = ephpubkey[:] @@ -395,7 +395,7 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey // Create session keys. sec := deriveKeys(sha256.New, ephkey, remotePubkey, c.localnode.ID(), challenge.Node.ID(), cdata) if sec == nil { - return nil, nil, fmt.Errorf("key derivation failed") + return nil, nil, errors.New("key derivation failed") } return auth, sec, err } diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index 8f1c221b8038..4f14d860e1ec 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -191,7 +191,7 @@ func (c *Client) resolveEntry(ctx context.Context, domain, hash string) (entry, func (c *Client) doResolveEntry(ctx context.Context, domain, hash string) (entry, error) { wantHash, err := b32format.DecodeString(hash) if err != nil { - return nil, fmt.Errorf("invalid base32 hash") + return nil, errors.New("invalid base32 hash") } name := hash + "." + domain txts, err := c.cfg.Resolver.LookupTXT(ctx, hash+"."+domain) diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index 7d9703a34558..dfac4fb37208 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -21,6 +21,7 @@ import ( "crypto/ecdsa" "encoding/base32" "encoding/base64" + "errors" "fmt" "io" "strings" @@ -341,7 +342,7 @@ func parseLinkEntry(e string) (entry, error) { func parseLink(e string) (*linkEntry, error) { if !strings.HasPrefix(e, linkPrefix) { - return nil, fmt.Errorf("wrong/missing scheme 'enrtree' in URL") + return nil, errors.New("wrong/missing scheme 'enrtree' in URL") } e = e[len(linkPrefix):] diff --git a/p2p/enode/idscheme.go b/p2p/enode/idscheme.go index fd5d868b761d..6ad7f809a71d 100644 --- a/p2p/enode/idscheme.go +++ b/p2p/enode/idscheme.go @@ -18,7 +18,7 @@ package enode import ( "crypto/ecdsa" - "fmt" + "errors" "io" "github.com/ethereum/go-ethereum/common/math" @@ -67,7 +67,7 @@ func (V4ID) Verify(r *enr.Record, sig []byte) error { if err := r.Load(&entry); err != nil { return err } else if len(entry) != 33 { - return fmt.Errorf("invalid public key") + return errors.New("invalid public key") } h := sha3.NewLegacyKeccak256() diff --git a/p2p/nat/natpmp.go b/p2p/nat/natpmp.go index 97601c99dcbb..ea2d8978293f 100644 --- a/p2p/nat/natpmp.go +++ b/p2p/nat/natpmp.go @@ -17,6 +17,7 @@ package nat import ( + "errors" "fmt" "net" "strings" @@ -46,7 +47,7 @@ func (n *pmp) ExternalIP() (net.IP, error) { func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { if lifetime <= 0 { - return 0, fmt.Errorf("lifetime must not be <= 0") + return 0, errors.New("lifetime must not be <= 0") } // Note order of port arguments is switched between our // AddMapping and the client's AddPortMapping. diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 63cc4936c1bd..17e0f75d5ab9 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -460,7 +460,7 @@ func startExecNodeStack() (*node.Node, error) { // decode the config confEnv := os.Getenv(envNodeConfig) if confEnv == "" { - return nil, fmt.Errorf("missing " + envNodeConfig) + return nil, errors.New("missing " + envNodeConfig) } var conf execNodeConfig if err := json.Unmarshal([]byte(confEnv), &conf); err != nil { diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 6bfcd2a727b4..e28f059106f3 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -708,7 +708,7 @@ func formatPrimitiveValue(encType string, encValue interface{}) (string, error) func (t Types) validate() error { for typeKey, typeArr := range t { if len(typeKey) == 0 { - return fmt.Errorf("empty type key") + return errors.New("empty type key") } for i, typeObj := range typeArr { if len(typeObj.Type) == 0 { diff --git a/trie/trie_test.go b/trie/trie_test.go index 379a866f7ea0..920594fdd24f 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -556,7 +556,7 @@ func runRandTest(rt randTest) error { checktr.MustUpdate(it.Key, it.Value) } if tr.Hash() != checktr.Hash() { - rt[i].err = fmt.Errorf("hash mismatch in opItercheckhash") + rt[i].err = errors.New("hash mismatch in opItercheckhash") } case opNodeDiff: var ( @@ -594,19 +594,19 @@ func runRandTest(rt randTest) error { } } if len(insertExp) != len(tr.tracer.inserts) { - rt[i].err = fmt.Errorf("insert set mismatch") + rt[i].err = errors.New("insert set mismatch") } if len(deleteExp) != len(tr.tracer.deletes) { - rt[i].err = fmt.Errorf("delete set mismatch") + rt[i].err = errors.New("delete set mismatch") } for insert := range tr.tracer.inserts { if _, present := insertExp[insert]; !present { - rt[i].err = fmt.Errorf("missing inserted node") + rt[i].err = errors.New("missing inserted node") } } for del := range tr.tracer.deletes { if _, present := deleteExp[del]; !present { - rt[i].err = fmt.Errorf("missing deleted node") + rt[i].err = errors.New("missing deleted node") } } } diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index 6e3f3faaedce..051e122bec64 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -215,7 +215,7 @@ func (m *meta) encode() []byte { // decode unpacks the meta object from byte stream. func (m *meta) decode(blob []byte) error { if len(blob) < 1 { - return fmt.Errorf("no version tag") + return errors.New("no version tag") } switch blob[0] { case stateHistoryVersion: From 865e1e9f577f4fa804d0246f82cbcedc27db9bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 29 Feb 2024 12:40:59 +0200 Subject: [PATCH 36/91] cmd/utils, core/rawdb, triedb/pathdb: flip hash to path scheme (#29108) * cmd/utils, core/rawdb, triedb/pathdb: flip hash to path scheme * graphql: run tests in hash mode as the chain maker needs it --- cmd/utils/flags.go | 3 +++ core/rawdb/accessors_trie.go | 8 +++----- graphql/graphql_test.go | 2 ++ triedb/pathdb/database.go | 1 - 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b813e52970d8..82af26ff9659 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1668,6 +1668,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 { cfg.TransactionHistory = 0 log.Warn("Disabled transaction unindexing for archive node") + + cfg.StateScheme = rawdb.HashScheme + log.Warn("Forcing hash state-scheme for archive mode") } if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index ea3367db3606..e34b24fd7661 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -315,7 +315,7 @@ func ReadStateScheme(db ethdb.Reader) string { // the stored state. // // - If the provided scheme is none, use the scheme consistent with persistent -// state, or fallback to hash-based scheme if state is empty. +// state, or fallback to path-based scheme if state is empty. // // - If the provided scheme is hash, use hash-based scheme or error out if not // compatible with persistent state scheme. @@ -329,10 +329,8 @@ func ParseStateScheme(provided string, disk ethdb.Database) (string, error) { stored := ReadStateScheme(disk) if provided == "" { if stored == "" { - // use default scheme for empty database, flip it when - // path mode is chosen as default - log.Info("State schema set to default", "scheme", "hash") - return HashScheme, nil + log.Info("State schema set to default", "scheme", "path") + return PathScheme, nil // use default scheme for empty database } log.Info("State scheme set to already existing", "scheme", stored) return stored, nil // reuse scheme of persistent scheme diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 1dda10205822..f3f9d1778ab0 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -452,6 +453,7 @@ func newGQLService(t *testing.T, stack *node.Node, shanghai bool, gspec *core.Ge TrieDirtyCache: 5, TrieTimeout: 60 * time.Minute, SnapshotCache: 5, + StateScheme: rawdb.HashScheme, } var engine consensus.Engine = ethash.NewFaker() if shanghai { diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 307f307df53d..3e8e83a00c2c 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -203,7 +203,6 @@ func New(diskdb ethdb.Database, config *Config) *Database { log.Crit("Failed to disable database", "err", err) // impossible to happen } } - log.Warn("Path-based state scheme is an experimental feature") return db } From 0a2f33946b95989e8ce36e72a88138adceab6a23 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:17:32 +0100 Subject: [PATCH 37/91] eth/catalyst: update simulated beacon for cancun (#28829) * eth/catalyst: update simulated beacon for cancun * validate blob hashes * compute hashes from commitment * fix beacon root and payload version * check commitment conversion * fix random attr * flip dev to cancun --- eth/catalyst/simulated_beacon.go | 21 ++++++++++++++++++--- params/config.go | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index f1c5689e1d28..4ae60ed4907c 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -18,6 +18,7 @@ package catalyst import ( "crypto/rand" + "crypto/sha256" "errors" "math/big" "sync" @@ -27,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -161,14 +163,14 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u SuggestedFeeRecipient: feeRecipient, Withdrawals: withdrawals, Random: random, - }, engine.PayloadV2, true) + BeaconRoot: &common.Hash{}, + }, engine.PayloadV3, true) if err != nil { return err } if fcResponse == engine.STATUS_SYNCING { return errors.New("chain rewind prevented invocation of payload creation") } - envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true) if err != nil { return err @@ -186,8 +188,21 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u } } + // Independently calculate the blob hashes from sidecars. + blobHashes := make([]common.Hash, 0) + if envelope.BlobsBundle != nil { + hasher := sha256.New() + for _, commit := range envelope.BlobsBundle.Commitments { + var c kzg4844.Commitment + if len(commit) != len(c) { + return errors.New("invalid commitment length") + } + copy(c[:], commit) + blobHashes = append(blobHashes, kzg4844.CalcBlobHashV1(hasher, &c)) + } + } // Mark the payload as canon - if _, err = c.engineAPI.NewPayloadV2(*payload); err != nil { + if _, err = c.engineAPI.NewPayloadV3(*payload, blobHashes, &common.Hash{}); err != nil { return err } c.setCurrentState(payload.BlockHash, finalizedHash) diff --git a/params/config.go b/params/config.go index 21ede457fd68..b24e782b8d96 100644 --- a/params/config.go +++ b/params/config.go @@ -183,6 +183,7 @@ var ( ArrowGlacierBlock: big.NewInt(0), GrayGlacierBlock: big.NewInt(0), ShanghaiTime: newUint64(0), + CancunTime: newUint64(0), TerminalTotalDifficulty: big.NewInt(0), TerminalTotalDifficultyPassed: true, } From 0b1438c3df5da5551e89dddc683d65f4d48ad3d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 2 Mar 2024 22:39:22 +0200 Subject: [PATCH 38/91] eth: make transaction propagation paths in the network deterministic (#29034) * eth: make transaction propagation paths in the network deterministic * eth: avoid potential division by 0 * eth: make tx propagation dependent on local node id too * eth: fix review comments --- eth/backend.go | 1 + eth/handler.go | 57 ++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index 0a0813aafac6..09e1dbd258cd 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -236,6 +236,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // Permit the downloader to use the trie cache allowance during fast sync cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit if eth.handler, err = newHandler(&handlerConfig{ + NodeID: eth.p2pServer.Self().ID(), Database: chainDb, Chain: eth.blockchain, TxPool: eth.txPool, diff --git a/eth/handler.go b/eth/handler.go index 0343a5787012..bc27eb4b88c5 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -41,7 +42,9 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/triedb/pathdb" + "golang.org/x/crypto/sha3" ) const ( @@ -84,6 +87,7 @@ type txPool interface { // handlerConfig is the collection of initialization parameters to create a full // node network handler. type handlerConfig struct { + NodeID enode.ID // P2P node ID used for tx propagation topology Database ethdb.Database // Database for direct sync insertions Chain *core.BlockChain // Blockchain to serve data from TxPool txPool // Transaction pool to propagate from @@ -96,6 +100,7 @@ type handlerConfig struct { } type handler struct { + nodeID enode.ID networkID uint64 forkFilter forkid.Filter // Fork ID filter, constant across the lifetime of the node @@ -137,6 +142,7 @@ func newHandler(config *handlerConfig) (*handler, error) { config.EventMux = new(event.TypeMux) // Nicety initialization for tests } h := &handler{ + nodeID: config.NodeID, networkID: config.Network, forkFilter: forkid.NewFilter(config.Chain), eventMux: config.EventMux, @@ -614,25 +620,54 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { annos = make(map[*ethPeer][]common.Hash) // Set peer->hash to announce ) // Broadcast transactions to a batch of peers not knowing about it - for _, tx := range txs { - peers := h.peers.peersWithoutTransaction(tx.Hash()) + direct := big.NewInt(int64(math.Sqrt(float64(h.peers.len())))) // Approximate number of peers to broadcast to + if direct.BitLen() == 0 { + direct = big.NewInt(1) + } + total := new(big.Int).Exp(direct, big.NewInt(2), nil) // Stabilise total peer count a bit based on sqrt peers - var numDirect int + var ( + signer = types.LatestSignerForChainID(h.chain.Config().ChainID) // Don't care about chain status, we just need *a* sender + hasher = sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash = make([]byte, 32) + ) + for _, tx := range txs { + var maybeDirect bool switch { case tx.Type() == types.BlobTxType: blobTxs++ case tx.Size() > txMaxBroadcastSize: largeTxs++ default: - numDirect = int(math.Sqrt(float64(len(peers)))) + maybeDirect = true } - // Send the tx unconditionally to a subset of our peers - for _, peer := range peers[:numDirect] { - txset[peer] = append(txset[peer], tx.Hash()) - } - // For the remaining peers, send announcement only - for _, peer := range peers[numDirect:] { - annos[peer] = append(annos[peer], tx.Hash()) + // Send the transaction (if it's small enough) directly to a subset of + // the peers that have not received it yet, ensuring that the flow of + // transactions is groupped by account to (try and) avoid nonce gaps. + // + // To do this, we hash the local enode IW with together with a peer's + // enode ID together with the transaction sender and broadcast if + // `sha(self, peer, sender) mod peers < sqrt(peers)`. + for _, peer := range h.peers.peersWithoutTransaction(tx.Hash()) { + var broadcast bool + if maybeDirect { + hasher.Reset() + hasher.Write(h.nodeID.Bytes()) + hasher.Write(peer.Node().ID().Bytes()) + + from, _ := types.Sender(signer, tx) // Ignore error, we only use the addr as a propagation target splitter + hasher.Write(from.Bytes()) + + hasher.Read(hash) + if new(big.Int).Mod(new(big.Int).SetBytes(hash), total).Cmp(direct) < 0 { + broadcast = true + } + } + if broadcast { + txset[peer] = append(txset[peer], tx.Hash()) + } else { + annos[peer] = append(annos[peer], tx.Hash()) + } } } for peer, hashes := range txset { From 00905f7dc406cfb67f64cd74113777044fb886d8 Mon Sep 17 00:00:00 2001 From: Undefinedor Date: Sun, 3 Mar 2024 04:42:50 +0800 Subject: [PATCH 39/91] all: remove redundant import aliases (#29144) --- cmd/geth/snapshot.go | 2 +- cmd/geth/verkle.go | 2 +- consensus/clique/clique.go | 2 +- eth/protocols/snap/metrics.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 1e0933e46f59..192c850868c8 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -36,7 +36,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - cli "github.com/urfave/cli/v2" + "github.com/urfave/cli/v2" ) var ( diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 420b063d8ba1..ff3931356e8f 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/gballet/go-verkle" - cli "github.com/urfave/cli/v2" + "github.com/urfave/cli/v2" ) var ( diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index c693189ea5ef..59f0e96ebe3e 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - lru "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" diff --git a/eth/protocols/snap/metrics.go b/eth/protocols/snap/metrics.go index a7d071953f67..a8dc2b582432 100644 --- a/eth/protocols/snap/metrics.go +++ b/eth/protocols/snap/metrics.go @@ -17,7 +17,7 @@ package snap import ( - metrics "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/metrics" ) var ( From a732ad036488e3d5db33928f0155ffd66e08c08d Mon Sep 17 00:00:00 2001 From: yzb Date: Mon, 4 Mar 2024 17:16:05 +0800 Subject: [PATCH 40/91] p2p: remove unused argument 'flags' (#29132) --- p2p/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/server.go b/p2p/server.go index 975a3bb91662..5b7afb4565b9 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -937,7 +937,7 @@ func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) c.transport = srv.newTransport(fd, dialDest.Pubkey()) } - err := srv.setupConn(c, flags, dialDest) + err := srv.setupConn(c, dialDest) if err != nil { if !c.is(inboundConn) { markDialError(err) @@ -947,7 +947,7 @@ func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) return err } -func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) error { +func (srv *Server) setupConn(c *conn, dialDest *enode.Node) error { // Prevent leftover pending conns from entering the handshake. srv.lock.Lock() running := srv.running From b408b3e5fece3524bf7721ac8dd8d9a898f571a8 Mon Sep 17 00:00:00 2001 From: yzb Date: Mon, 4 Mar 2024 17:24:24 +0800 Subject: [PATCH 41/91] accounts/abi: delete duplicate error check (#29136) --- accounts/abi/type.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/accounts/abi/type.go b/accounts/abi/type.go index 2eee11787fda..383982663331 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -179,9 +179,6 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty return Type{}, errors.New("abi: purely anonymous or underscored field is not supported") } fieldName := ResolveNameConflict(name, func(s string) bool { return used[s] }) - if err != nil { - return Type{}, err - } used[fieldName] = true if !isValidFieldName(fieldName) { return Type{}, fmt.Errorf("field %d has invalid name", idx) From 5a1e8a6547d6606c7ff1e3f3841fbb1c9f205282 Mon Sep 17 00:00:00 2001 From: cui <523516579@qq.com> Date: Mon, 4 Mar 2024 17:30:15 +0800 Subject: [PATCH 42/91] core: delete unused ErrMaxInitCodeSizeExceeded (#29062) --- core/vm/errors.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/vm/errors.go b/core/vm/errors.go index fbbf19e178bf..004f8ef1c83c 100644 --- a/core/vm/errors.go +++ b/core/vm/errors.go @@ -29,7 +29,6 @@ var ( ErrInsufficientBalance = errors.New("insufficient balance for transfer") ErrContractAddressCollision = errors.New("contract address collision") ErrExecutionReverted = errors.New("execution reverted") - ErrMaxInitCodeSizeExceeded = errors.New("max initcode size exceeded") ErrMaxCodeSizeExceeded = errors.New("max code size exceeded") ErrInvalidJump = errors.New("invalid jump destination") ErrWriteProtection = errors.New("write protection") From 679a27a2b36d4f86e6b49c49c0d51c47a7ef6145 Mon Sep 17 00:00:00 2001 From: buddho Date: Mon, 4 Mar 2024 17:31:18 +0800 Subject: [PATCH 43/91] all: use EmptyUncleHash, EmptyCodeHash instead of raw value (#29134) --- cmd/devp2p/internal/ethtest/snap.go | 2 +- core/types/block_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index 64e063358545..8ff3f1f71a6e 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -648,7 +648,7 @@ The server should reject the request.`, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8}}, }, nBytes: 5000, - expHashes: []common.Hash{common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")}, + expHashes: []common.Hash{types.EmptyCodeHash}, }, { diff --git a/core/types/block_test.go b/core/types/block_test.go index cf0b1dd85c1e..982d002242f6 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -196,7 +196,7 @@ func TestEIP2718BlockEncoding(t *testing.T) { func TestUncleHash(t *testing.T) { uncles := make([]*Header, 0) h := CalcUncleHash(uncles) - exp := common.HexToHash("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347") + exp := EmptyUncleHash if h != exp { t.Fatalf("empty uncle hash is wrong, got %x != %x", h, exp) } From 35cebc16877c4cfbf48b883ab3bfa02b9100a87a Mon Sep 17 00:00:00 2001 From: psogv0308 Date: Mon, 4 Mar 2024 19:03:53 +0900 Subject: [PATCH 44/91] triedb/pathdb: changed the test code to check for verifying state (#29150) Co-authored-by: this-is-iron --- triedb/pathdb/database_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index e7bd4699938d..df69942e9aa2 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -397,7 +397,11 @@ func TestDatabaseRollback(t *testing.T) { if err := tester.db.Recover(parent, loader); err != nil { t.Fatalf("Failed to revert db, err: %v", err) } - tester.verifyState(parent) + if i > 0 { + if err := tester.verifyState(parent); err != nil { + t.Fatalf("Failed to verify state, err: %v", err) + } + } } if tester.db.tree.len() != 1 { t.Fatal("Only disk layer is expected") From a97d622588c2b71557c6222b95d487f51b46bd78 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 4 Mar 2024 14:07:41 +0100 Subject: [PATCH 45/91] cmd/devp2p: fix commandHasFlag (#29091) It got broken in some update of the cli library, and thus bootnodes weren't being configured automatically for some of the discovery commands. --- cmd/devp2p/main.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/devp2p/main.go b/cmd/devp2p/main.go index 8461a8b9b5e2..66974bba5890 100644 --- a/cmd/devp2p/main.go +++ b/cmd/devp2p/main.go @@ -66,9 +66,15 @@ func commandHasFlag(ctx *cli.Context, flag cli.Flag) bool { for _, name := range names { set[name] = struct{}{} } - for _, fn := range ctx.FlagNames() { - if _, ok := set[fn]; ok { - return true + for _, ctx := range ctx.Lineage() { + if ctx.Command != nil { + for _, f := range ctx.Command.Flags { + for _, name := range f.Names() { + if _, ok := set[name]; ok { + return true + } + } + } } } return false From ca473b81cbe4a96cde4e8424c49b15ab304787bb Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 4 Mar 2024 22:25:53 +0800 Subject: [PATCH 46/91] core: use finalized block as the chain freeze indicator (#28683) * core: use finalized block as the chain freeze indicator * core/rawdb: use max(finality, head-90k) as chain freezing threshold * core/rawdb: fix tests * core/rawdb: fix lint * core/rawdb: address comments from peter * core/rawdb: fix typo --- core/blockchain_repair_test.go | 10 ++- core/blockchain_sethead_test.go | 8 ++- core/rawdb/chain_freezer.go | 113 ++++++++++++++++++++------------ core/rawdb/database.go | 8 +-- 4 files changed, 84 insertions(+), 55 deletions(-) diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index b2df39d17bb8..b6a299f8ba89 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -1757,7 +1757,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) { // It's hard to follow the test case, visualize the input - //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // fmt.Println(tt.dump(true)) // Create a temporary persistent database @@ -1830,10 +1830,14 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s } // Force run a freeze cycle type freezer interface { - Freeze(threshold uint64) error + Freeze() error Ancients() (uint64, error) } - db.(freezer).Freeze(tt.freezeThreshold) + if tt.freezeThreshold < uint64(tt.canonicalBlocks) { + final := uint64(tt.canonicalBlocks) - tt.freezeThreshold + chain.SetFinalized(canonblocks[int(final)-1].Header()) + } + db.(freezer).Freeze() // Set the simulated pivot block if tt.pivotBlock != nil { diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 1504c74e0ef3..b96ee12c9952 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -2044,10 +2044,14 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme // Force run a freeze cycle type freezer interface { - Freeze(threshold uint64) error + Freeze() error Ancients() (uint64, error) } - db.(freezer).Freeze(tt.freezeThreshold) + if tt.freezeThreshold < uint64(tt.canonicalBlocks) { + final := uint64(tt.canonicalBlocks) - tt.freezeThreshold + chain.SetFinalized(canonblocks[int(final)-1].Header()) + } + db.(freezer).Freeze() // Set the simulated pivot block if tt.pivotBlock != nil { diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index bb2c409dbbd5..d8214874bdb8 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -17,9 +17,9 @@ package rawdb import ( + "errors" "fmt" "sync" - "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -43,8 +43,6 @@ const ( // The background thread will keep moving ancient chain segments from key-value // database to flat files for saving space on live database. type chainFreezer struct { - threshold atomic.Uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests) - *Freezer quit chan struct{} wg sync.WaitGroup @@ -57,13 +55,11 @@ func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFre if err != nil { return nil, err } - cf := chainFreezer{ + return &chainFreezer{ Freezer: freezer, quit: make(chan struct{}), trigger: make(chan chan struct{}), - } - cf.threshold.Store(params.FullImmutabilityThreshold) - return &cf, nil + }, nil } // Close closes the chain freezer instance and terminates the background thread. @@ -77,6 +73,57 @@ func (f *chainFreezer) Close() error { return f.Freezer.Close() } +// readHeadNumber returns the number of chain head block. 0 is returned if the +// block is unknown or not available yet. +func (f *chainFreezer) readHeadNumber(db ethdb.KeyValueReader) uint64 { + hash := ReadHeadBlockHash(db) + if hash == (common.Hash{}) { + log.Error("Head block is not reachable") + return 0 + } + number := ReadHeaderNumber(db, hash) + if number == nil { + log.Error("Number of head block is missing") + return 0 + } + return *number +} + +// readFinalizedNumber returns the number of finalized block. 0 is returned +// if the block is unknown or not available yet. +func (f *chainFreezer) readFinalizedNumber(db ethdb.KeyValueReader) uint64 { + hash := ReadFinalizedBlockHash(db) + if hash == (common.Hash{}) { + return 0 + } + number := ReadHeaderNumber(db, hash) + if number == nil { + log.Error("Number of finalized block is missing") + return 0 + } + return *number +} + +// freezeThreshold returns the threshold for chain freezing. It's determined +// by formula: max(finality, HEAD-params.FullImmutabilityThreshold). +func (f *chainFreezer) freezeThreshold(db ethdb.KeyValueReader) (uint64, error) { + var ( + head = f.readHeadNumber(db) + final = f.readFinalizedNumber(db) + headLimit uint64 + ) + if head > params.FullImmutabilityThreshold { + headLimit = head - params.FullImmutabilityThreshold + } + if final == 0 && headLimit == 0 { + return 0, errors.New("freezing threshold is not available") + } + if final > headLimit { + return final, nil + } + return headLimit, nil +} + // freeze is a background thread that periodically checks the blockchain for any // import progress and moves ancient data from the fast database into the freezer. // @@ -114,60 +161,39 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { return } } - // Retrieve the freezing threshold. - hash := ReadHeadBlockHash(nfdb) - if hash == (common.Hash{}) { - log.Debug("Current full block hash unavailable") // new chain, empty database + threshold, err := f.freezeThreshold(nfdb) + if err != nil { backoff = true + log.Debug("Current full block not old enough to freeze", "err", err) continue } - number := ReadHeaderNumber(nfdb, hash) - threshold := f.threshold.Load() frozen := f.frozen.Load() - switch { - case number == nil: - log.Error("Current full block number unavailable", "hash", hash) - backoff = true - continue - case *number < threshold: - log.Debug("Current full block not old enough to freeze", "number", *number, "hash", hash, "delay", threshold) - backoff = true - continue - - case *number-threshold <= frozen: - log.Debug("Ancient blocks frozen already", "number", *number, "hash", hash, "frozen", frozen) + // Short circuit if the blocks below threshold are already frozen. + if frozen != 0 && frozen-1 >= threshold { backoff = true + log.Debug("Ancient blocks frozen already", "threshold", threshold, "frozen", frozen) continue } - head := ReadHeader(nfdb, hash, *number) - if head == nil { - log.Error("Current full block unavailable", "number", *number, "hash", hash) - backoff = true - continue - } - // Seems we have data ready to be frozen, process in usable batches var ( - start = time.Now() - first, _ = f.Ancients() - limit = *number - threshold + start = time.Now() + first = frozen // the first block to freeze + last = threshold // the last block to freeze ) - if limit-first > freezerBatchLimit { - limit = first + freezerBatchLimit + if last-first+1 > freezerBatchLimit { + last = freezerBatchLimit + first - 1 } - ancients, err := f.freezeRange(nfdb, first, limit) + ancients, err := f.freezeRange(nfdb, first, last) if err != nil { log.Error("Error in block freeze operation", "err", err) backoff = true continue } - // Batch of blocks have been frozen, flush them before wiping from leveldb if err := f.Sync(); err != nil { log.Crit("Failed to flush frozen tables", "err", err) } - // Wipe out all data from the active database batch := db.NewBatch() for i := 0; i < len(ancients); i++ { @@ -250,8 +276,11 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { } } +// freezeRange moves a batch of chain segments from the fast database to the freezer. +// The parameters (number, limit) specify the relevant block range, both of which +// are included. func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes []common.Hash, err error) { - hashes = make([]common.Hash, 0, limit-number) + hashes = make([]common.Hash, 0, limit-number+1) _, err = f.ModifyAncients(func(op ethdb.AncientWriteOp) error { for ; number <= limit; number++ { @@ -293,11 +322,9 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash if err := op.AppendRaw(ChainFreezerDifficultyTable, number, td); err != nil { return fmt.Errorf("can't write td to Freezer: %v", err) } - hashes = append(hashes, hash) } return nil }) - return hashes, err } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 27a9ec7412ca..9cab30bfcd1f 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -66,16 +66,10 @@ func (frdb *freezerdb) Close() error { // Freeze is a helper method used for external testing to trigger and block until // a freeze cycle completes, without having to sleep for a minute to trigger the // automatic background run. -func (frdb *freezerdb) Freeze(threshold uint64) error { +func (frdb *freezerdb) Freeze() error { if frdb.AncientStore.(*chainFreezer).readonly { return errReadOnly } - // Set the freezer threshold to a temporary value - defer func(old uint64) { - frdb.AncientStore.(*chainFreezer).threshold.Store(old) - }(frdb.AncientStore.(*chainFreezer).threshold.Load()) - frdb.AncientStore.(*chainFreezer).threshold.Store(threshold) - // Trigger a freeze cycle and block until it's done trigger := make(chan struct{}, 1) frdb.AncientStore.(*chainFreezer).trigger <- trigger From 19607d1a10d37542ba13ab9db48cf4e501715cce Mon Sep 17 00:00:00 2001 From: Andrei Silviu Dragnea Date: Mon, 4 Mar 2024 20:21:43 +0100 Subject: [PATCH 47/91] eth/tracers: Fix prestateTracer pre nonce on contract creation (#29099) The prestateTracer was reporting an inaccurate nonce for the contract being created in post EIP-158 transactions. Correct nonce is 0, due to the issue nonce was being reported as 1. --- .../prestate_tracer/create_post_eip158.json | 64 +++++++++++++++ .../create_post_eip158.json | 82 +++++++++++++++++++ eth/tracers/native/prestate.go | 3 + 3 files changed, 149 insertions(+) create mode 100644 eth/tracers/internal/tracetest/testdata/prestate_tracer/create_post_eip158.json create mode 100644 eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_post_eip158.json diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer/create_post_eip158.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer/create_post_eip158.json new file mode 100644 index 000000000000..205b472dabe4 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer/create_post_eip158.json @@ -0,0 +1,64 @@ +{ + "genesis": { + "baseFeePerGas": "7", + "difficulty": "2", + "extraData": "0xd983010d0e846765746888676f312e32312e318664617277696e0000000000001713699f05f79a59abec177c7a87b90ceda79b72ff5edc9197dd7627a447cde45b079bbc3765a236cdf680e2d4d2247135d0e6bb6fd92b50638b92504ddb274f00", + "gasLimit": "30000000", + "hash": "0x6ad5258175c66f4e883d238a92a08428d8ebcbeac631ab7b972634cc05effab3", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "39137", + "stateRoot": "0x715f00df764dbadd4863247a215ac44b5420beafde3ec458b15db7aafa89be0c", + "timestamp": "1709022192", + "totalDifficulty": "78275", + "alloc": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": "2" + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": "64" + } + }, + "config": { + "chainId": 12345, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "clique": { + "period": 5, + "epoch": 30000 + } + } + }, + "context": { + "number": "39138", + "difficulty": "2", + "timestamp": "1709022197", + "gasLimit": "30000000", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0" + }, + "input": "0x02f902af823039408459682f008459682f088302b3538080b90254608060405234801561001057600080fd5b50610234806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806309ce9ccb1461003b5780633fb5c1cb14610059575b600080fd5b610043610075565b60405161005091906100e2565b60405180910390f35b610073600480360381019061006e919061012e565b61007b565b005b60005481565b80600081905550600a8111156100c6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100bd906101de565b60405180910390fd5b50565b6000819050919050565b6100dc816100c9565b82525050565b60006020820190506100f760008301846100d3565b92915050565b600080fd5b61010b816100c9565b811461011657600080fd5b50565b60008135905061012881610102565b92915050565b600060208284031215610144576101436100fd565b5b600061015284828501610119565b91505092915050565b600082825260208201905092915050565b7f4e756d6265722069732067726561746572207468616e2031302c207472616e7360008201527f616374696f6e2072657665727465642e00000000000000000000000000000000602082015250565b60006101c860308361015b565b91506101d38261016c565b604082019050919050565b600060208201905081810360008301526101f7816101bb565b905091905056fea264697066735822122069018995fecf03bda91a88b6eafe41641709dee8b4a706fe301c8a569fe8c1b364736f6c63430008130033c001a0a8cf4729b7e4664687abb3e2559853d7d489eb441519be2a17493061fb4c3a03a04b5a904ba8a6e59c6c40049c4d14a73233aeb8a45b38403199f304630dc0d453", + "result": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": 2 + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": 64 + } + } +} diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_post_eip158.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_post_eip158.json new file mode 100644 index 000000000000..83266f6669cf --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_post_eip158.json @@ -0,0 +1,82 @@ +{ + "genesis": { + "baseFeePerGas": "7", + "difficulty": "2", + "extraData": "0xd983010d0e846765746888676f312e32312e318664617277696e0000000000001713699f05f79a59abec177c7a87b90ceda79b72ff5edc9197dd7627a447cde45b079bbc3765a236cdf680e2d4d2247135d0e6bb6fd92b50638b92504ddb274f00", + "gasLimit": "30000000", + "hash": "0x6ad5258175c66f4e883d238a92a08428d8ebcbeac631ab7b972634cc05effab3", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "39137", + "stateRoot": "0x715f00df764dbadd4863247a215ac44b5420beafde3ec458b15db7aafa89be0c", + "timestamp": "1709022192", + "totalDifficulty": "78275", + "alloc": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": "2" + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": "64" + } + }, + "config": { + "chainId": 12345, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "clique": { + "period": 5, + "epoch": 30000 + } + } + }, + "context": { + "number": "39138", + "difficulty": "2", + "timestamp": "1709022197", + "gasLimit": "30000000", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0" + }, + "input": "0x02f902af823039408459682f008459682f088302b3538080b90254608060405234801561001057600080fd5b50610234806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806309ce9ccb1461003b5780633fb5c1cb14610059575b600080fd5b610043610075565b60405161005091906100e2565b60405180910390f35b610073600480360381019061006e919061012e565b61007b565b005b60005481565b80600081905550600a8111156100c6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100bd906101de565b60405180910390fd5b50565b6000819050919050565b6100dc816100c9565b82525050565b60006020820190506100f760008301846100d3565b92915050565b600080fd5b61010b816100c9565b811461011657600080fd5b50565b60008135905061012881610102565b92915050565b600060208284031215610144576101436100fd565b5b600061015284828501610119565b91505092915050565b600082825260208201905092915050565b7f4e756d6265722069732067726561746572207468616e2031302c207472616e7360008201527f616374696f6e2072657665727465642e00000000000000000000000000000000602082015250565b60006101c860308361015b565b91506101d38261016c565b604082019050919050565b600060208201905081810360008301526101f7816101bb565b905091905056fea264697066735822122069018995fecf03bda91a88b6eafe41641709dee8b4a706fe301c8a569fe8c1b364736f6c63430008130033c001a0a8cf4729b7e4664687abb3e2559853d7d489eb441519be2a17493061fb4c3a03a04b5a904ba8a6e59c6c40049c4d14a73233aeb8a45b38403199f304630dc0d453", + "tracerConfig": { + "diffMode": true + }, + "result": { + "post": { + "0x1bda2f8e4735507930bd6cfe873bf0bf0f4ab1de": { + "code": "0x608060405234801561001057600080fd5b50600436106100365760003560e01c806309ce9ccb1461003b5780633fb5c1cb14610059575b600080fd5b610043610075565b60405161005091906100e2565b60405180910390f35b610073600480360381019061006e919061012e565b61007b565b005b60005481565b80600081905550600a8111156100c6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100bd906101de565b60405180910390fd5b50565b6000819050919050565b6100dc816100c9565b82525050565b60006020820190506100f760008301846100d3565b92915050565b600080fd5b61010b816100c9565b811461011657600080fd5b50565b60008135905061012881610102565b92915050565b600060208284031215610144576101436100fd565b5b600061015284828501610119565b91505092915050565b600082825260208201905092915050565b7f4e756d6265722069732067726561746572207468616e2031302c207472616e7360008201527f616374696f6e2072657665727465642e00000000000000000000000000000000602082015250565b60006101c860308361015b565b91506101d38261016c565b604082019050919050565b600060208201905081810360008301526101f7816101bb565b905091905056fea264697066735822122069018995fecf03bda91a88b6eafe41641709dee8b4a706fe301c8a569fe8c1b364736f6c63430008130033", + "nonce": 1 + }, + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f0645688331eb5690" + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6aae9b21b6ee855", + "nonce": 65 + } + }, + "pre": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": 2 + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": 64 + } + } + } +} diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index d7e10173cf27..0d57f62caf20 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -102,6 +102,9 @@ func (t *prestateTracer) CaptureStart(env *vm.EVM, from common.Address, to commo // The recipient balance includes the value transferred. toBal := new(big.Int).Sub(t.pre[to].Balance, value) t.pre[to].Balance = toBal + if env.ChainConfig().Rules(env.Context.BlockNumber, env.Context.Random != nil, env.Context.Time).IsEIP158 && create { + t.pre[to].Nonce-- + } // The sender balance is after reducing: value and gasLimit. // We need to re-add them to get the pre-tx balance. From 5d5b384efd0acabe4d808c46fce9700114d2046f Mon Sep 17 00:00:00 2001 From: Domino Valdano Date: Mon, 4 Mar 2024 12:58:25 -0800 Subject: [PATCH 48/91] .mailmap: remove invalid email address (#29163) --- .mailmap | 1 - 1 file changed, 1 deletion(-) diff --git a/.mailmap b/.mailmap index aa074b76d6b5..312e51d8544f 100644 --- a/.mailmap +++ b/.mailmap @@ -56,7 +56,6 @@ Diederik Loerakker Dimitry Khokhlov Domino Valdano -Domino Valdano Edgar Aroutiounian From 9b3ceb2137df125dd0f6957a362e9f08d6c41b66 Mon Sep 17 00:00:00 2001 From: Vie Date: Tue, 5 Mar 2024 15:33:52 +0800 Subject: [PATCH 49/91] core/types: reuse signtx (#29152) * core/types: reuse signtx * core/types: inline signtx --- core/types/transaction_signing.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 9e26642f753d..70dee0776e7b 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -107,13 +107,7 @@ func SignTx(tx *Transaction, s Signer, prv *ecdsa.PrivateKey) (*Transaction, err // SignNewTx creates a transaction and signs it. func SignNewTx(prv *ecdsa.PrivateKey, s Signer, txdata TxData) (*Transaction, error) { - tx := NewTx(txdata) - h := s.Hash(tx) - sig, err := crypto.Sign(h[:], prv) - if err != nil { - return nil, err - } - return tx.WithSignature(s, sig) + return SignTx(NewTx(txdata), s, prv) } // MustSignNewTx creates a transaction and signs it. From d89d7ebdec27d8c8fed217767e2f17b09b5460a0 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 5 Mar 2024 16:47:58 +0800 Subject: [PATCH 50/91] core: initialize `gasRemaining` with `=` instead of `+=` (#29149) initialize gasRemaining with = instead of += --- core/state_transition.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state_transition.go b/core/state_transition.go index 9c4f76d1c585..bda2a995bf31 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -263,7 +263,7 @@ func (st *StateTransition) buyGas() error { if err := st.gp.SubGas(st.msg.GasLimit); err != nil { return err } - st.gasRemaining += st.msg.GasLimit + st.gasRemaining = st.msg.GasLimit st.initialGas = st.msg.GasLimit mgvalU256, _ := uint256.FromBig(mgval) From e199319fd680aa4b135147f0480549a1c7d95350 Mon Sep 17 00:00:00 2001 From: buddho Date: Tue, 5 Mar 2024 17:47:56 +0800 Subject: [PATCH 51/91] rlp: remove a moot todo (#29154) --- rlp/iterator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/rlp/iterator.go b/rlp/iterator.go index 6be574572e61..95bd3f258208 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -23,7 +23,6 @@ type listIterator struct { } // NewListIterator creates an iterator for the (list) represented by data -// TODO: Consider removing this implementation, as it is no longer used. func NewListIterator(data RawValue) (*listIterator, error) { k, t, c, err := readKind(data) if err != nil { From 7b81cf6362b3bb52762b823edf2a31bbbed4aa84 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 5 Mar 2024 21:31:55 +0800 Subject: [PATCH 52/91] core/state, trie/triedb/pathdb: remove storage incomplete flag (#28940) As SELF-DESTRUCT opcode is disabled in the cancun fork(unless the account is created within the same transaction, nothing to delete in this case). The account will only be deleted in the following cases: - The account is created within the same transaction. In this case the original storage was empty. - The account is empty(zero nonce, zero balance, zero code) and is touched within the transaction. Fortunately this kind of accounts are not-existent on ethereum-mainnet. All in all, after cancun, we are pretty sure there is no large contract deletion and we don't need this mechanism for oom protection. --- core/state/metrics.go | 1 - core/state/statedb.go | 87 ++++++++++++---------------------- core/state/statedb_test.go | 4 +- trie/triestate/state.go | 15 +++--- triedb/pathdb/database.go | 3 -- triedb/pathdb/database_test.go | 4 +- triedb/pathdb/disklayer.go | 7 --- triedb/pathdb/history.go | 40 +++++----------- triedb/pathdb/history_test.go | 2 +- triedb/pathdb/journal.go | 32 ++++++------- 10 files changed, 67 insertions(+), 128 deletions(-) diff --git a/core/state/metrics.go b/core/state/metrics.go index 64c651461e73..7447e44dfd1e 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -33,5 +33,4 @@ var ( slotDeletionTimer = metrics.NewRegisteredResettingTimer("state/delete/storage/timer", nil) slotDeletionCount = metrics.NewRegisteredMeter("state/delete/storage/slot", nil) slotDeletionSize = metrics.NewRegisteredMeter("state/delete/storage/size", nil) - slotDeletionSkip = metrics.NewRegisteredGauge("state/delete/storage/skip", nil) ) diff --git a/core/state/statedb.go b/core/state/statedb.go index a4b8cf93e2d2..4d1163d3c6af 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -36,12 +36,6 @@ import ( "github.com/holiman/uint256" ) -const ( - // storageDeleteLimit denotes the highest permissible memory allocation - // employed for contract storage deletion. - storageDeleteLimit = 512 * 1024 * 1024 -) - type revision struct { id int journalIndex int @@ -949,10 +943,10 @@ func (s *StateDB) clearJournalAndRefund() { // of a specific account. It leverages the associated state snapshot for fast // storage iteration and constructs trie node deletion markers by creating // stack trie with iterated slots. -func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { iter, err := s.snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{}) if err != nil { - return false, 0, nil, nil, err + return 0, nil, nil, err } defer iter.Release() @@ -968,40 +962,37 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo }) stack := trie.NewStackTrie(options) for iter.Next() { - if size > storageDeleteLimit { - return true, size, nil, nil, nil - } slot := common.CopyBytes(iter.Slot()) if err := iter.Error(); err != nil { // error might occur after Slot function - return false, 0, nil, nil, err + return 0, nil, nil, err } size += common.StorageSize(common.HashLength + len(slot)) slots[iter.Hash()] = slot if err := stack.Update(iter.Hash().Bytes(), slot); err != nil { - return false, 0, nil, nil, err + return 0, nil, nil, err } } if err := iter.Error(); err != nil { // error might occur during iteration - return false, 0, nil, nil, err + return 0, nil, nil, err } if stack.Hash() != root { - return false, 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) + return 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) } - return false, size, slots, nodes, nil + return size, slots, nodes, nil } // slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage," // employed when the associated state snapshot is not available. It iterates the // storage slots along with all internal trie nodes via trie directly. -func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie) if err != nil { - return false, 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) + return 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) } it, err := tr.NodeIterator(nil) if err != nil { - return false, 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) + return 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) } var ( size common.StorageSize @@ -1009,9 +1000,6 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r slots = make(map[common.Hash][]byte) ) for it.Next(true) { - if size > storageDeleteLimit { - return true, size, nil, nil, nil - } if it.Leaf() { slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob()) size += common.StorageSize(common.HashLength + len(it.LeafBlob())) @@ -1024,9 +1012,9 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r nodes.AddNode(it.Path(), trienode.NewDeleted()) } if err := it.Error(); err != nil { - return false, 0, nil, nil, err + return 0, nil, nil, err } - return false, size, slots, nodes, nil + return size, slots, nodes, nil } // deleteStorage is designed to delete the storage trie of a designated account. @@ -1034,31 +1022,27 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r // potentially leading to an out-of-memory panic. The function will make an attempt // to utilize an efficient strategy if the associated state snapshot is reachable; // otherwise, it will resort to a less-efficient approach. -func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { var ( - start = time.Now() - err error - aborted bool - size common.StorageSize - slots map[common.Hash][]byte - nodes *trienode.NodeSet + start = time.Now() + err error + size common.StorageSize + slots map[common.Hash][]byte + nodes *trienode.NodeSet ) // The fast approach can be failed if the snapshot is not fully // generated, or it's internally corrupted. Fallback to the slow // one just in case. if s.snap != nil { - aborted, size, slots, nodes, err = s.fastDeleteStorage(addrHash, root) + size, slots, nodes, err = s.fastDeleteStorage(addrHash, root) } if s.snap == nil || err != nil { - aborted, size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) + size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) } if err != nil { - return false, nil, nil, err + return nil, nil, err } if metrics.EnabledExpensive { - if aborted { - slotDeletionSkip.Inc(1) - } n := int64(len(slots)) slotDeletionMaxCount.UpdateIfGt(int64(len(slots))) @@ -1068,7 +1052,7 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root slotDeletionCount.Mark(n) slotDeletionSize.Mark(int64(size)) } - return aborted, slots, nodes, nil + return slots, nodes, nil } // handleDestruction processes all destruction markers and deletes the account @@ -1095,13 +1079,12 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root // // In case (d), **original** account along with its storages should be deleted, // with their values be tracked as original value. -func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.Address]struct{}, error) { +func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) error { // Short circuit if geth is running with hash mode. This procedure can consume // considerable time and storage deletion isn't supported in hash mode, thus // preemptively avoiding unnecessary expenses. - incomplete := make(map[common.Address]struct{}) if s.db.TrieDB().Scheme() == rawdb.HashScheme { - return incomplete, nil + return nil } for addr, prev := range s.stateObjectsDestruct { // The original account was non-existing, and it's marked as destructed @@ -1124,18 +1107,9 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A continue } // Remove storage slots belong to the account. - aborted, slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) + slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) if err != nil { - return nil, fmt.Errorf("failed to delete storage, err: %w", err) - } - // The storage is too huge to handle, skip it but mark as incomplete. - // For case (d), the account is resurrected might with a few slots - // created. In this case, wipe the entire storage state diff because - // of aborted deletion. - if aborted { - incomplete[addr] = struct{}{} - delete(s.storagesOrigin, addr) - continue + return fmt.Errorf("failed to delete storage, err: %w", err) } if s.storagesOrigin[addr] == nil { s.storagesOrigin[addr] = slots @@ -1147,10 +1121,10 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A } } if err := nodes.Merge(set); err != nil { - return nil, err + return err } } - return incomplete, nil + return nil } // Commit writes the state to the underlying in-memory trie database. @@ -1179,8 +1153,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er codeWriter = s.db.DiskDB().NewBatch() ) // Handle all state deletions first - incomplete, err := s.handleDestruction(nodes) - if err != nil { + if err := s.handleDestruction(nodes); err != nil { return common.Hash{}, err } // Handle all state updates afterwards @@ -1276,7 +1249,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er } if root != origin { start := time.Now() - set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete) + set := triestate.New(s.accountsOrigin, s.storagesOrigin) if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil { return common.Hash{}, err } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index cd86a7f4b67f..3649b0ac589b 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -1161,12 +1161,12 @@ func TestDeleteStorage(t *testing.T) { obj := fastState.getOrNewStateObject(addr) storageRoot := obj.data.Root - _, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } - _, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } diff --git a/trie/triestate/state.go b/trie/triestate/state.go index 4c47e9c39734..aa4d32f852f9 100644 --- a/trie/triestate/state.go +++ b/trie/triestate/state.go @@ -59,18 +59,16 @@ type TrieLoader interface { // The value refers to the original content of state before the transition // is made. Nil means that the state was not present previously. type Set struct { - Accounts map[common.Address][]byte // Mutated account set, nil means the account was not present - Storages map[common.Address]map[common.Hash][]byte // Mutated storage set, nil means the slot was not present - Incomplete map[common.Address]struct{} // Indicator whether the storage is incomplete due to large deletion - size common.StorageSize // Approximate size of set + Accounts map[common.Address][]byte // Mutated account set, nil means the account was not present + Storages map[common.Address]map[common.Hash][]byte // Mutated storage set, nil means the slot was not present + size common.StorageSize // Approximate size of set } // New constructs the state set with provided data. -func New(accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte, incomplete map[common.Address]struct{}) *Set { +func New(accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte) *Set { return &Set{ - Accounts: accounts, - Storages: storages, - Incomplete: incomplete, + Accounts: accounts, + Storages: storages, } } @@ -88,7 +86,6 @@ func (s *Set) Size() common.StorageSize { } s.size += common.StorageSize(common.AddressLength) } - s.size += common.StorageSize(common.AddressLength * len(s.Incomplete)) return s.size } diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 3e8e83a00c2c..b1e01abac4d6 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -403,9 +403,6 @@ func (db *Database) Recoverable(root common.Hash) bool { if m.parent != root { return errors.New("unexpected state history") } - if len(m.incomplete) > 0 { - return errors.New("incomplete state history") - } root = m.root return nil }) == nil diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index df69942e9aa2..2e7e1bef05ff 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -299,7 +299,7 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode } } } - return root, ctx.nodes, triestate.New(ctx.accountOrigin, ctx.storageOrigin, nil) + return root, ctx.nodes, triestate.New(ctx.accountOrigin, ctx.storageOrigin) } // lastRoot returns the latest root hash, or empty if nothing is cached. @@ -543,7 +543,7 @@ func TestCorruptedJournal(t *testing.T) { // Mutate the journal in disk, it should be regarded as invalid blob := rawdb.ReadTrieJournal(tester.db.diskdb) - blob[0] = 1 + blob[0] = 0xa rawdb.WriteTrieJournal(tester.db.diskdb, blob) // Verify states, all not-yet-written states should be discarded diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index ef697cbce8ce..777e4ec8a750 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -17,7 +17,6 @@ package pathdb import ( - "errors" "fmt" "sync" @@ -239,12 +238,6 @@ func (dl *diskLayer) revert(h *history, loader triestate.TrieLoader) (*diskLayer if h.meta.root != dl.rootHash() { return nil, errUnexpectedHistory } - // Reject if the provided state history is incomplete. It's due to - // a large construct SELF-DESTRUCT which can't be handled because - // of memory limitation. - if len(h.meta.incomplete) > 0 { - return nil, errors.New("incomplete state history") - } if dl.id == 0 { return nil, fmt.Errorf("%w: zero state id", errStateUnrecoverable) } diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index 051e122bec64..68fb4809f01d 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -66,7 +66,7 @@ import ( const ( accountIndexSize = common.AddressLength + 13 // The length of encoded account index slotIndexSize = common.HashLength + 5 // The length of encoded slot index - historyMetaSize = 9 + 2*common.HashLength // The length of fixed size part of meta object + historyMetaSize = 9 + 2*common.HashLength // The length of encoded history meta stateHistoryVersion = uint8(0) // initial version of state history structure. ) @@ -192,23 +192,19 @@ func (i *slotIndex) decode(blob []byte) { // meta describes the meta data of state history object. type meta struct { - version uint8 // version tag of history object - parent common.Hash // prev-state root before the state transition - root common.Hash // post-state root after the state transition - block uint64 // associated block number - incomplete []common.Address // list of address whose storage set is incomplete + version uint8 // version tag of history object + parent common.Hash // prev-state root before the state transition + root common.Hash // post-state root after the state transition + block uint64 // associated block number } // encode packs the meta object into byte stream. func (m *meta) encode() []byte { - buf := make([]byte, historyMetaSize+len(m.incomplete)*common.AddressLength) + buf := make([]byte, historyMetaSize) buf[0] = m.version copy(buf[1:1+common.HashLength], m.parent.Bytes()) copy(buf[1+common.HashLength:1+2*common.HashLength], m.root.Bytes()) binary.BigEndian.PutUint64(buf[1+2*common.HashLength:historyMetaSize], m.block) - for i, h := range m.incomplete { - copy(buf[i*common.AddressLength+historyMetaSize:], h.Bytes()) - } return buf[:] } @@ -219,20 +215,13 @@ func (m *meta) decode(blob []byte) error { } switch blob[0] { case stateHistoryVersion: - if len(blob) < historyMetaSize { + if len(blob) != historyMetaSize { return fmt.Errorf("invalid state history meta, len: %d", len(blob)) } - if (len(blob)-historyMetaSize)%common.AddressLength != 0 { - return fmt.Errorf("corrupted state history meta, len: %d", len(blob)) - } m.version = blob[0] m.parent = common.BytesToHash(blob[1 : 1+common.HashLength]) m.root = common.BytesToHash(blob[1+common.HashLength : 1+2*common.HashLength]) m.block = binary.BigEndian.Uint64(blob[1+2*common.HashLength : historyMetaSize]) - for pos := historyMetaSize; pos < len(blob); { - m.incomplete = append(m.incomplete, common.BytesToAddress(blob[pos:pos+common.AddressLength])) - pos += common.AddressLength - } return nil default: return fmt.Errorf("unknown version %d", blob[0]) @@ -257,7 +246,6 @@ func newHistory(root common.Hash, parent common.Hash, block uint64, states *trie var ( accountList []common.Address storageList = make(map[common.Address][]common.Hash) - incomplete []common.Address ) for addr := range states.Accounts { accountList = append(accountList, addr) @@ -272,18 +260,12 @@ func newHistory(root common.Hash, parent common.Hash, block uint64, states *trie slices.SortFunc(slist, common.Hash.Cmp) storageList[addr] = slist } - for addr := range states.Incomplete { - incomplete = append(incomplete, addr) - } - slices.SortFunc(incomplete, common.Address.Cmp) - return &history{ meta: &meta{ - version: stateHistoryVersion, - parent: parent, - root: root, - block: block, - incomplete: incomplete, + version: stateHistoryVersion, + parent: parent, + root: root, + block: block, }, accounts: states.Accounts, accountList: accountList, diff --git a/triedb/pathdb/history_test.go b/triedb/pathdb/history_test.go index a3257441de80..ab0d44777d7b 100644 --- a/triedb/pathdb/history_test.go +++ b/triedb/pathdb/history_test.go @@ -47,7 +47,7 @@ func randomStateSet(n int) *triestate.Set { account := generateAccount(types.EmptyRootHash) accounts[addr] = types.SlimAccountRLP(account) } - return triestate.New(accounts, storages, nil) + return triestate.New(accounts, storages) } func makeHistory() *history { diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index ac770763e38d..3a0b7ebae273 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -41,7 +41,13 @@ var ( errUnmatchedJournal = errors.New("unmatched journal") ) -const journalVersion uint64 = 0 +// journalVersion ensures that an incompatible journal is detected and discarded. +// +// Changelog: +// +// - Version 0: initial version +// - Version 1: storage.Incomplete field is removed +const journalVersion uint64 = 1 // journalNode represents a trie node persisted in the journal. type journalNode struct { @@ -64,10 +70,9 @@ type journalAccounts struct { // journalStorage represents a list of storage slots belong to an account. type journalStorage struct { - Incomplete bool - Account common.Address - Hashes []common.Hash - Slots [][]byte + Account common.Address + Hashes []common.Hash + Slots [][]byte } // loadJournal tries to parse the layer journal from the disk. @@ -209,11 +214,10 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) { } // Read state changes from journal var ( - jaccounts journalAccounts - jstorages []journalStorage - accounts = make(map[common.Address][]byte) - storages = make(map[common.Address]map[common.Hash][]byte) - incomplete = make(map[common.Address]struct{}) + jaccounts journalAccounts + jstorages []journalStorage + accounts = make(map[common.Address][]byte) + storages = make(map[common.Address]map[common.Hash][]byte) ) if err := r.Decode(&jaccounts); err != nil { return nil, fmt.Errorf("load diff accounts: %v", err) @@ -233,12 +237,9 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) { set[h] = nil } } - if entry.Incomplete { - incomplete[entry.Account] = struct{}{} - } storages[entry.Account] = set } - return db.loadDiffLayer(newDiffLayer(parent, root, parent.stateID()+1, block, nodes, triestate.New(accounts, storages, incomplete)), r) + return db.loadDiffLayer(newDiffLayer(parent, root, parent.stateID()+1, block, nodes, triestate.New(accounts, storages)), r) } // journal implements the layer interface, marshaling the un-flushed trie nodes @@ -316,9 +317,6 @@ func (dl *diffLayer) journal(w io.Writer) error { storage := make([]journalStorage, 0, len(dl.states.Storages)) for addr, slots := range dl.states.Storages { entry := journalStorage{Account: addr} - if _, ok := dl.states.Incomplete[addr]; ok { - entry.Incomplete = true - } for slotHash, slot := range slots { entry.Hashes = append(entry.Hashes, slotHash) entry.Slots = append(entry.Slots, slot) From 96bf23f1ea95d29a32abe8fe2992b86e892b6c4c Mon Sep 17 00:00:00 2001 From: Martin HS Date: Tue, 5 Mar 2024 14:32:47 +0100 Subject: [PATCH 53/91] accounts/usbwallet: use updated hid (only) library (#28945) * accounts/usbwallet: use updated hid (only) library * deps: update karalabe/hid --- accounts/usbwallet/hub.go | 8 ++++---- accounts/usbwallet/wallet.go | 6 +++--- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/accounts/usbwallet/hub.go b/accounts/usbwallet/hub.go index e67942dbc107..e22dffe9718e 100644 --- a/accounts/usbwallet/hub.go +++ b/accounts/usbwallet/hub.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" - "github.com/karalabe/usb" + "github.com/karalabe/hid" ) // LedgerScheme is the protocol scheme prefixing account and wallet URLs. @@ -109,7 +109,7 @@ func NewTrezorHubWithWebUSB() (*Hub, error) { // newHub creates a new hardware wallet manager for generic USB devices. func newHub(scheme string, vendorID uint16, productIDs []uint16, usageID uint16, endpointID int, makeDriver func(log.Logger) driver) (*Hub, error) { - if !usb.Supported() { + if !hid.Supported() { return nil, errors.New("unsupported platform") } hub := &Hub{ @@ -155,7 +155,7 @@ func (hub *Hub) refreshWallets() { return } // Retrieve the current list of USB wallet devices - var devices []usb.DeviceInfo + var devices []hid.DeviceInfo if runtime.GOOS == "linux" { // hidapi on Linux opens the device during enumeration to retrieve some infos, @@ -170,7 +170,7 @@ func (hub *Hub) refreshWallets() { return } } - infos, err := usb.Enumerate(hub.vendorID, 0) + infos, err := hid.Enumerate(hub.vendorID, 0) if err != nil { failcount := hub.enumFails.Add(1) if runtime.GOOS == "linux" { diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 69083dc8939d..0fd0415a9ef8 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/karalabe/usb" + "github.com/karalabe/hid" ) // Maximum time between wallet health checks to detect USB unplugs. @@ -79,8 +79,8 @@ type wallet struct { driver driver // Hardware implementation of the low level device operations url *accounts.URL // Textual URL uniquely identifying this wallet - info usb.DeviceInfo // Known USB device infos about the wallet - device usb.Device // USB device advertising itself as a hardware wallet + info hid.DeviceInfo // Known USB device infos about the wallet + device hid.Device // USB device advertising itself as a hardware wallet accounts []accounts.Account // List of derive accounts pinned on the hardware wallet paths map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations diff --git a/go.mod b/go.mod index 7a54b1ff7ca9..48faa0a321d3 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 github.com/julienschmidt/httprouter v1.3.0 - github.com/karalabe/usb v0.0.2 + github.com/karalabe/hid v1.0.1-0.20240209121748-d3b59fe37df8 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.17 diff --git a/go.sum b/go.sum index bb4ded5c2ff7..20a95d36877c 100644 --- a/go.sum +++ b/go.sum @@ -385,8 +385,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= -github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/hid v1.0.1-0.20240209121748-d3b59fe37df8 h1:IMoiklsIksD2ii43zKCybVU6jLNzpSl3bT31+5mUjgg= +github.com/karalabe/hid v1.0.1-0.20240209121748-d3b59fe37df8/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= From dfa6c5e9c80e0965d0476909afc26e87aa199e6a Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 5 Mar 2024 21:37:26 +0800 Subject: [PATCH 54/91] internal/jsre: format blob fields from hexdecimal to int (#29166) * internal/jsre: format receipt.{blobGasPrice,blobGasUsed} to int Signed-off-by: jsvisa * internal/jsre: format tx.maxFeePerBlobGas to int Signed-off-by: jsvisa * internal/jsre: format blob* in block Signed-off-by: jsvisa --------- Signed-off-by: jsvisa --- internal/jsre/deps/web3.js | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index 0b360e74150a..4196cb8db0ee 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -3734,7 +3734,7 @@ var inputCallFormatter = function (options){ options.to = inputAddressFormatter(options.to); } - ['maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { + ['maxFeePerBlobGas', 'maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { return options[key] !== undefined; }).forEach(function(key){ options[key] = utils.fromDecimal(options[key]); @@ -3759,7 +3759,7 @@ var inputTransactionFormatter = function (options){ options.to = inputAddressFormatter(options.to); } - ['maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { + ['maxFeePerBlobGas', 'maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { return options[key] !== undefined; }).forEach(function(key){ options[key] = utils.fromDecimal(options[key]); @@ -3789,6 +3789,9 @@ var outputTransactionFormatter = function (tx){ if(tx.maxPriorityFeePerGas !== undefined) { tx.maxPriorityFeePerGas = utils.toBigNumber(tx.maxPriorityFeePerGas); } + if(tx.maxFeePerBlobGas !== undefined) { + tx.maxFeePerBlobGas = utils.toBigNumber(tx.maxFeePerBlobGas); + } tx.value = utils.toBigNumber(tx.value); return tx; }; @@ -3810,6 +3813,12 @@ var outputTransactionReceiptFormatter = function (receipt){ if(receipt.effectiveGasPrice !== undefined) { receipt.effectiveGasPrice = utils.toBigNumber(receipt.effectiveGasPrice); } + if(receipt.blobGasPrice !== undefined) { + receipt.blobGasPrice = utils.toBigNumber(receipt.blobGasPrice); + } + if(receipt.blobGasUsed !== undefined) { + receipt.blobGasUsed = utils.toBigNumber(receipt.blobGasUsed); + } if(utils.isArray(receipt.logs)) { receipt.logs = receipt.logs.map(function(log){ return outputLogFormatter(log); @@ -3831,11 +3840,17 @@ var outputBlockFormatter = function(block) { if (block.baseFeePerGas !== undefined) { block.baseFeePerGas = utils.toBigNumber(block.baseFeePerGas); } + if (block.blobGasUsed !== undefined) { + block.blobGasUsed = utils.toBigNumber(block.blobGasUsed); + } + if (block.excessBlobGas !== undefined) { + block.excessBlobGas = utils.toBigNumber(block.excessBlobGas); + } block.gasLimit = utils.toDecimal(block.gasLimit); block.gasUsed = utils.toDecimal(block.gasUsed); block.size = utils.toDecimal(block.size); block.timestamp = utils.toDecimal(block.timestamp); - if(block.number !== null) + if (block.number !== null) block.number = utils.toDecimal(block.number); block.difficulty = utils.toBigNumber(block.difficulty); From a6d6e8ac410170eb1085b9e7b0388b1c67f95548 Mon Sep 17 00:00:00 2001 From: Undefinedor Date: Tue, 5 Mar 2024 21:44:23 +0800 Subject: [PATCH 55/91] rpc: remove deprecated method "Notifier.Closed" (#29162) --- eth/downloader/api.go | 2 -- eth/filters/api.go | 6 ------ eth/tracers/api.go | 4 ++-- node/api.go | 2 -- p2p/simulations/http_test.go | 2 -- rpc/subscription.go | 6 ------ rpc/testservice_test.go | 5 +---- 7 files changed, 3 insertions(+), 24 deletions(-) diff --git a/eth/downloader/api.go b/eth/downloader/api.go index 6b8cb98e23bd..90c36afbb5ba 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -149,8 +149,6 @@ func (api *DownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error notifier.Notify(rpcSub.ID, status) case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() diff --git a/eth/filters/api.go b/eth/filters/api.go index 8cf701ec5713..59103ac03ca6 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -179,8 +179,6 @@ func (api *FilterAPI) NewPendingTransactions(ctx context.Context, fullTx *bool) } case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() @@ -241,8 +239,6 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { notifier.Notify(rpcSub.ID, h) case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() @@ -278,8 +274,6 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc } case <-rpcSub.Err(): // client send an unsubscribe request return - case <-notifier.Closed(): // connection dropped - return } } }() diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 683310820526..fa8c881d1a71 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -226,7 +226,7 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf } sub := notifier.CreateSubscription() - resCh := api.traceChain(from, to, config, notifier.Closed()) + resCh := api.traceChain(from, to, config, sub.Err()) go func() { for result := range resCh { notifier.Notify(sub.ID, result) @@ -240,7 +240,7 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf // the end block but excludes the start one. The return value will be one item per // transaction, dependent on the requested tracer. // The tracing procedure should be aborted in case the closed signal is received. -func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed <-chan interface{}) chan *blockTraceResult { +func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed <-chan error) chan *blockTraceResult { reexec := defaultTraceReexec if config != nil && config.Reexec != nil { reexec = *config.Reexec diff --git a/node/api.go b/node/api.go index f81f394beb24..a71ae6aa2954 100644 --- a/node/api.go +++ b/node/api.go @@ -145,8 +145,6 @@ func (api *adminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, error) return case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index c53a49797bd3..c04308fe0bf8 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -282,8 +282,6 @@ func (t *TestAPI) Events(ctx context.Context) (*rpc.Subscription, error) { return case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() diff --git a/rpc/subscription.go b/rpc/subscription.go index 9cb07275479e..d3dff32a272e 100644 --- a/rpc/subscription.go +++ b/rpc/subscription.go @@ -145,12 +145,6 @@ func (n *Notifier) Notify(id ID, data any) error { return nil } -// Closed returns a channel that is closed when the RPC connection is closed. -// Deprecated: use subscription error channel -func (n *Notifier) Closed() <-chan interface{} { - return n.h.conn.closed() -} - // takeSubscription returns the subscription (if one has been created). No subscription can // be created after this call. func (n *Notifier) takeSubscription() *Subscription { diff --git a/rpc/testservice_test.go b/rpc/testservice_test.go index 7d873af6670e..69199e21b711 100644 --- a/rpc/testservice_test.go +++ b/rpc/testservice_test.go @@ -195,10 +195,7 @@ func (s *notificationTestService) SomeSubscription(ctx context.Context, n, val i return } } - select { - case <-notifier.Closed(): - case <-subscription.Err(): - } + <-subscription.Err() if s.unsubscribed != nil { s.unsubscribed <- string(subscription.ID) } From a970295956d602c348dccce034712c14aedce5e0 Mon Sep 17 00:00:00 2001 From: cui <523516579@qq.com> Date: Tue, 5 Mar 2024 21:45:17 +0800 Subject: [PATCH 56/91] rlp: using unsafe.Slice instead of SliceHeader (#29067) Co-authored-by: Felix Lange --- rlp/unsafe.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/rlp/unsafe.go b/rlp/unsafe.go index 2152ba35fc4a..10868caaf287 100644 --- a/rlp/unsafe.go +++ b/rlp/unsafe.go @@ -26,10 +26,5 @@ import ( // byteArrayBytes returns a slice of the byte array v. func byteArrayBytes(v reflect.Value, length int) []byte { - var s []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&s)) - hdr.Data = v.UnsafeAddr() - hdr.Cap = length - hdr.Len = length - return s + return unsafe.Slice((*byte)(unsafe.Pointer(v.UnsafeAddr())), length) } From 9e129efd7b43242fb5e605065713c27d615e753d Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 5 Mar 2024 21:48:27 +0800 Subject: [PATCH 57/91] core: remove useless assignments (#29065) --- core/state_transition.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index bda2a995bf31..8fcf4c093dbc 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -234,7 +234,7 @@ func (st *StateTransition) to() common.Address { func (st *StateTransition) buyGas() error { mgval := new(big.Int).SetUint64(st.msg.GasLimit) - mgval = mgval.Mul(mgval, st.msg.GasPrice) + mgval.Mul(mgval, st.msg.GasPrice) balanceCheck := new(big.Int).Set(mgval) if st.msg.GasFeeCap != nil { balanceCheck.SetUint64(st.msg.GasLimit) @@ -477,7 +477,7 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { // Return ETH for remaining gas, exchanged at the original rate. remaining := uint256.NewInt(st.gasRemaining) - remaining = remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice)) + remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice)) st.state.AddBalance(st.msg.From, remaining) // Also return remaining gas to the block gas counter so it is From 9a0fa8093ca5f7b896c3f7e849f7ca532d24e2a6 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 5 Mar 2024 14:52:44 +0100 Subject: [PATCH 58/91] node: remove test which doesn't do a lot (#29159) * node: fix test if directory already exists * node: remove test --- node/node_test.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/node/node_test.go b/node/node_test.go index 04810a815bf6..d1d1e5dfe8fa 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -415,21 +415,6 @@ func TestRegisterHandler_Successful(t *testing.T) { assert.Equal(t, "success", string(buf)) } -// Tests that the given handler will not be successfully mounted since no HTTP server -// is enabled for RPC -func TestRegisterHandler_Unsuccessful(t *testing.T) { - node, err := New(&DefaultConfig) - if err != nil { - t.Fatalf("could not create new node: %v", err) - } - - // create and mount handler - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("success")) - }) - node.RegisterHandler("test", "/test", handler) -} - // Tests whether websocket requests can be handled on the same port as a regular http server. func TestWebsocketHTTPOnSamePort_WebsocketRequest(t *testing.T) { node := startHTTP(t, 0, 0) From f4d53133f6e4b13f0dbcfef3bc45e9650d863b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 5 Mar 2024 16:13:28 +0200 Subject: [PATCH 59/91] consensus, cmd, core, eth: remove support for non-merge mode of operation (#29169) * eth: drop support for forward sync triggers and head block packets * consensus, eth: enforce always merged network * eth: fix tx looper startup and shutdown * cmd, core: fix some tests * core: remove notion of future blocks * core, eth: drop unused methods and types --- cmd/geth/testdata/clique.json | 1 + consensus/merger.go | 110 ---- core/block_validator_test.go | 6 - core/blockchain.go | 108 +--- core/blockchain_insert.go | 5 - core/blockchain_test.go | 5 - eth/backend.go | 34 +- eth/catalyst/api.go | 45 +- eth/catalyst/api_test.go | 4 - eth/ethconfig/config.go | 13 +- eth/fetcher/block_fetcher.go | 939 ----------------------------- eth/fetcher/block_fetcher_test.go | 949 ------------------------------ eth/fetcher/tx_fetcher.go | 10 +- eth/handler.go | 180 +----- eth/handler_eth.go | 62 -- eth/handler_eth_test.go | 159 ----- eth/handler_test.go | 2 - eth/peerset.go | 34 -- eth/protocols/eth/broadcast.go | 33 -- eth/protocols/eth/handlers.go | 37 +- eth/protocols/eth/peer.go | 107 +--- eth/protocols/eth/protocol.go | 13 - eth/sync.go | 215 ------- eth/sync_test.go | 5 +- params/config.go | 2 + 25 files changed, 88 insertions(+), 2990 deletions(-) delete mode 100644 consensus/merger.go delete mode 100644 eth/fetcher/block_fetcher.go delete mode 100644 eth/fetcher/block_fetcher_test.go diff --git a/cmd/geth/testdata/clique.json b/cmd/geth/testdata/clique.json index b54b4a7d3b72..36f5c3105703 100644 --- a/cmd/geth/testdata/clique.json +++ b/cmd/geth/testdata/clique.json @@ -8,6 +8,7 @@ "byzantiumBlock": 0, "constantinopleBlock": 0, "petersburgBlock": 0, + "terminalTotalDifficultyPassed": true, "clique": { "period": 5, "epoch": 30000 diff --git a/consensus/merger.go b/consensus/merger.go deleted file mode 100644 index ffbcbf2b8569..000000000000 --- a/consensus/merger.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2021 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package consensus - -import ( - "fmt" - "sync" - - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" -) - -// transitionStatus describes the status of eth1/2 transition. This switch -// between modes is a one-way action which is triggered by corresponding -// consensus-layer message. -type transitionStatus struct { - LeftPoW bool // The flag is set when the first NewHead message received - EnteredPoS bool // The flag is set when the first FinalisedBlock message received -} - -// Merger is an internal help structure used to track the eth1/2 transition status. -// It's a common structure can be used in both full node and light client. -type Merger struct { - db ethdb.KeyValueStore - status transitionStatus - mu sync.RWMutex -} - -// NewMerger creates a new Merger which stores its transition status in the provided db. -func NewMerger(db ethdb.KeyValueStore) *Merger { - var status transitionStatus - blob := rawdb.ReadTransitionStatus(db) - if len(blob) != 0 { - if err := rlp.DecodeBytes(blob, &status); err != nil { - log.Crit("Failed to decode the transition status", "err", err) - } - } - return &Merger{ - db: db, - status: status, - } -} - -// ReachTTD is called whenever the first NewHead message received -// from the consensus-layer. -func (m *Merger) ReachTTD() { - m.mu.Lock() - defer m.mu.Unlock() - - if m.status.LeftPoW { - return - } - m.status = transitionStatus{LeftPoW: true} - blob, err := rlp.EncodeToBytes(m.status) - if err != nil { - panic(fmt.Sprintf("Failed to encode the transition status: %v", err)) - } - rawdb.WriteTransitionStatus(m.db, blob) - log.Info("Left PoW stage") -} - -// FinalizePoS is called whenever the first FinalisedBlock message received -// from the consensus-layer. -func (m *Merger) FinalizePoS() { - m.mu.Lock() - defer m.mu.Unlock() - - if m.status.EnteredPoS { - return - } - m.status = transitionStatus{LeftPoW: true, EnteredPoS: true} - blob, err := rlp.EncodeToBytes(m.status) - if err != nil { - panic(fmt.Sprintf("Failed to encode the transition status: %v", err)) - } - rawdb.WriteTransitionStatus(m.db, blob) - log.Info("Entered PoS stage") -} - -// TDDReached reports whether the chain has left the PoW stage. -func (m *Merger) TDDReached() bool { - m.mu.RLock() - defer m.mu.RUnlock() - - return m.status.LeftPoW -} - -// PoSFinalized reports whether the chain has entered the PoS stage. -func (m *Merger) PoSFinalized() bool { - m.mu.RLock() - defer m.mu.RUnlock() - - return m.status.EnteredPoS -} diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 385c0afd9d40..2f86b2d751b8 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -94,7 +94,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { preBlocks []*types.Block postBlocks []*types.Block engine consensus.Engine - merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) ) if isClique { var ( @@ -186,11 +185,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { } chain.InsertChain(preBlocks[i : i+1]) } - - // Make the transition - merger.ReachTTD() - merger.FinalizePoS() - // Verify the blocks after the merging for i := 0; i < len(postBlocks); i++ { _, results := engine.VerifyHeaders(chain, []*types.Header{postHeaders[i]}) diff --git a/core/blockchain.go b/core/blockchain.go index b1bbc3d5982d..67b49cfe0262 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -50,7 +50,6 @@ import ( "github.com/ethereum/go-ethereum/triedb" "github.com/ethereum/go-ethereum/triedb/hashdb" "github.com/ethereum/go-ethereum/triedb/pathdb" - "golang.org/x/exp/slices" ) var ( @@ -97,13 +96,11 @@ var ( ) const ( - bodyCacheLimit = 256 - blockCacheLimit = 256 - receiptsCacheLimit = 32 - txLookupCacheLimit = 1024 - maxFutureBlocks = 256 - maxTimeFutureBlocks = 30 - TriesInMemory = 128 + bodyCacheLimit = 256 + blockCacheLimit = 256 + receiptsCacheLimit = 32 + txLookupCacheLimit = 1024 + TriesInMemory = 128 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. // @@ -245,9 +242,6 @@ type BlockChain struct { blockCache *lru.Cache[common.Hash, *types.Block] txLookupCache *lru.Cache[common.Hash, txLookup] - // future blocks are blocks added for later processing - futureBlocks *lru.Cache[common.Hash, *types.Block] - wg sync.WaitGroup quit chan struct{} // shutdown signal, closed in Stop. stopping atomic.Bool // false if chain is running, true when stopped @@ -299,7 +293,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), - futureBlocks: lru.NewCache[common.Hash, *types.Block](maxFutureBlocks), engine: engine, vmConfig: vmConfig, } @@ -449,11 +442,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis } bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root) } - - // Start future block processor. - bc.wg.Add(1) - go bc.updateFutureBlocks() - // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) @@ -794,7 +782,6 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha bc.receiptsCache.Purge() bc.blockCache.Purge() bc.txLookupCache.Purge() - bc.futureBlocks.Purge() // Clear safe block, finalized block if needed if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.Number.Uint64() { @@ -1048,24 +1035,6 @@ func (bc *BlockChain) insertStopped() bool { return bc.procInterrupt.Load() } -func (bc *BlockChain) procFutureBlocks() { - blocks := make([]*types.Block, 0, bc.futureBlocks.Len()) - for _, hash := range bc.futureBlocks.Keys() { - if block, exist := bc.futureBlocks.Peek(hash); exist { - blocks = append(blocks, block) - } - } - if len(blocks) > 0 { - slices.SortFunc(blocks, func(a, b *types.Block) int { - return a.Number().Cmp(b.Number()) - }) - // Insert one by one as chain insertion needs contiguous ancestry between blocks - for i := range blocks { - bc.InsertChain(blocks[i : i+1]) - } - } -} - // WriteStatus status of write type WriteStatus byte @@ -1466,8 +1435,6 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types if status == CanonStatTy { bc.writeHeadBlock(block) } - bc.futureBlocks.Remove(block.Hash()) - if status == CanonStatTy { bc.chainFeed.Send(ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) if len(logs) > 0 { @@ -1487,25 +1454,6 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types return status, nil } -// addFutureBlock checks if the block is within the max allowed window to get -// accepted for future processing, and returns an error if the block is too far -// ahead and was not added. -// -// TODO after the transition, the future block shouldn't be kept. Because -// it's not checked in the Geth side anymore. -func (bc *BlockChain) addFutureBlock(block *types.Block) error { - max := uint64(time.Now().Unix() + maxTimeFutureBlocks) - if block.Time() > max { - return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max) - } - if block.Difficulty().Cmp(common.Big0) == 0 { - // Never add PoS blocks into the future queue - return nil - } - bc.futureBlocks.Add(block.Hash(), block) - return nil -} - // InsertChain attempts to insert the given batch of blocks in to the canonical // chain or, otherwise, create a fork. If an error is returned it will return // the index number of the failing block as well an error describing what went @@ -1643,26 +1591,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) _, err := bc.recoverAncestors(block) return it.index, err } - // First block is future, shove it (and all children) to the future queue (unknown ancestor) - case errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash())): - for block != nil && (it.index == 0 || errors.Is(err, consensus.ErrUnknownAncestor)) { - log.Debug("Future block, postponing import", "number", block.Number(), "hash", block.Hash()) - if err := bc.addFutureBlock(block); err != nil { - return it.index, err - } - block, err = it.next() - } - stats.queued += it.processed() - stats.ignored += it.remaining() - - // If there are any still remaining, mark as ignored - return it.index, err - // Some other error(except ErrKnownBlock) occurred, abort. // ErrKnownBlock is allowed here since some known blocks // still need re-execution to generate snapshots that are missing case err != nil && !errors.Is(err, ErrKnownBlock): - bc.futureBlocks.Remove(block.Hash()) stats.ignored += len(it.chain) bc.reportBlock(block, nil, err) return it.index, err @@ -1867,23 +1799,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) "root", block.Root()) } } - - // Any blocks remaining here? The only ones we care about are the future ones - if block != nil && errors.Is(err, consensus.ErrFutureBlock) { - if err := bc.addFutureBlock(block); err != nil { - return it.index, err - } - block, err = it.next() - - for ; block != nil && errors.Is(err, consensus.ErrUnknownAncestor); block, err = it.next() { - if err := bc.addFutureBlock(block); err != nil { - return it.index, err - } - stats.queued++ - } - } stats.ignored += it.remaining() - return it.index, err } @@ -2334,20 +2250,6 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) { return head.Hash(), nil } -func (bc *BlockChain) updateFutureBlocks() { - futureTimer := time.NewTicker(5 * time.Second) - defer futureTimer.Stop() - defer bc.wg.Done() - for { - select { - case <-futureTimer.C: - bc.procFutureBlocks() - case <-bc.quit: - return - } - } -} - // skipBlock returns 'true', if the block being imported can be skipped over, meaning // that the block does not need to be processed but can be considered already fully 'done'. func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool { diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index 9bf662b6b710..c7c4c4bfea8b 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -179,8 +179,3 @@ func (it *insertIterator) first() *types.Block { func (it *insertIterator) remaining() int { return len(it.chain) - it.index } - -// processed returns the number of processed blocks. -func (it *insertIterator) processed() int { - return it.index + 1 -} diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 876d662f74d8..4fa759129c9a 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -2129,7 +2129,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate a canonical chain to act as the main dataset chainConfig := *params.TestChainConfig var ( - merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) engine = beacon.New(ethash.NewFaker()) key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key.PublicKey) @@ -2153,8 +2152,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Activate the transition since genesis if required if mergePoint == 0 { mergeBlock = 0 - merger.ReachTTD() - merger.FinalizePoS() // Set the terminal total difficulty in the config gspec.Config.TerminalTotalDifficulty = big.NewInt(0) @@ -2189,8 +2186,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Activate the transition in the middle of the chain if mergePoint == 1 { - merger.ReachTTD() - merger.FinalizePoS() // Set the terminal total difficulty in the config ttd := big.NewInt(int64(len(blocks))) ttd.Mul(ttd, params.GenesisDifficulty) diff --git a/eth/backend.go b/eth/backend.go index 09e1dbd258cd..f6c1637acadf 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -74,7 +74,6 @@ type Ethereum struct { handler *handler ethDialCandidates enode.Iterator snapDialCandidates enode.Iterator - merger *consensus.Merger // DB interfaces chainDb ethdb.Database // Block chain database @@ -158,7 +157,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } eth := &Ethereum{ config: config, - merger: consensus.NewMerger(chainDb), chainDb: chainDb, eventMux: stack.EventMux(), accountManager: stack.AccountManager(), @@ -240,7 +238,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { Database: chainDb, Chain: eth.blockchain, TxPool: eth.txPool, - Merger: eth.merger, Network: networkID, Sync: config.SyncMode, BloomCache: uint64(cacheLimit), @@ -487,11 +484,6 @@ func (s *Ethereum) Synced() bool { return s.handler.synced func (s *Ethereum) SetSynced() { s.handler.enableSyncedFeatures() } func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning } func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } -func (s *Ethereum) Merger() *consensus.Merger { return s.merger } -func (s *Ethereum) SyncMode() downloader.SyncMode { - mode, _ := s.handler.chainSync.modeAndLocalHead() - return mode -} // Protocols returns all the currently configured // network protocols to start. @@ -551,3 +543,29 @@ func (s *Ethereum) Stop() error { return nil } + +// SyncMode retrieves the current sync mode, either explicitly set, or derived +// from the chain status. +func (s *Ethereum) SyncMode() downloader.SyncMode { + // If we're in snap sync mode, return that directly + if s.handler.snapSync.Load() { + return downloader.SnapSync + } + // We are probably in full sync, but we might have rewound to before the + // snap sync pivot, check if we should re-enable snap sync. + head := s.blockchain.CurrentBlock() + if pivot := rawdb.ReadLastPivotNumber(s.chainDb); pivot != nil { + if head.Number.Uint64() < *pivot { + return downloader.SnapSync + } + } + // We are in a full sync, but the associated head state is missing. To complete + // the head state, forcefully rerun the snap sync. Note it doesn't mean the + // persistent state is corrupted, just mismatch with the head block. + if !s.blockchain.HasState(head.Root) { + log.Info("Reenabled snap sync as chain is stateless") + return downloader.SnapSync + } + // Nope, we're really full syncing + return downloader.FullSync +} diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index fea9d34cb83e..e5781b2c8f3e 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -267,12 +267,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl finalized := api.remoteBlocks.get(update.FinalizedBlockHash) // Header advertised via a past newPayload request. Start syncing to it. - // Before we do however, make sure any legacy sync in switched off so we - // don't accidentally have 2 cycles running. - if merger := api.eth.Merger(); !merger.TDDReached() { - merger.ReachTTD() - api.eth.Downloader().Cancel() - } context := []interface{}{"number", header.Number, "hash", header.Hash()} if update.FinalizedBlockHash != (common.Hash{}) { if finalized == nil { @@ -334,9 +328,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl // If the beacon client also advertised a finalized block, mark the local // chain final and completely in PoS mode. if update.FinalizedBlockHash != (common.Hash{}) { - if merger := api.eth.Merger(); !merger.PoSFinalized() { - merger.FinalizePoS() - } // If the finalized block is not in our canonical tree, something is wrong finalBlock := api.eth.BlockChain().GetBlockByHash(update.FinalizedBlockHash) if finalBlock == nil { @@ -620,13 +611,6 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe return api.invalid(err, parent.Header()), nil } - // We've accepted a valid payload from the beacon client. Mark the local - // chain transitions to notify other subsystems (e.g. downloader) of the - // behavioral change. - if merger := api.eth.Merger(); !merger.TDDReached() { - merger.ReachTTD() - api.eth.Downloader().Cancel() - } hash := block.Hash() return engine.PayloadStatusV1{Status: engine.VALID, LatestValidHash: &hash}, nil } @@ -784,26 +768,23 @@ func (api *ConsensusAPI) heartbeat() { // If there have been no updates for the past while, warn the user // that the beacon client is probably offline - if api.eth.BlockChain().Config().TerminalTotalDifficultyPassed || api.eth.Merger().TDDReached() { - if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout { - offlineLogged = time.Time{} - continue - } - - if time.Since(offlineLogged) > beaconUpdateWarnFrequency { - if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { - if lastTransitionUpdate.IsZero() { - log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") - } else { - log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") - } + if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout { + offlineLogged = time.Time{} + continue + } + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { + if lastTransitionUpdate.IsZero() { + log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") } else { - log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!") + log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") } - offlineLogged = time.Now() + } else { + log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!") } - continue + offlineLogged = time.Now() } + continue } } diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index cc1258ca55bf..a82e2d6cf6f2 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -862,7 +862,6 @@ func TestTrickRemoteBlockCache(t *testing.T) { func TestInvalidBloom(t *testing.T) { genesis, preMergeBlocks := generateMergeChain(10, false) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() commonAncestor := ethservice.BlockChain().CurrentBlock() @@ -1044,7 +1043,6 @@ func TestWithdrawals(t *testing.T) { genesis.Config.ShanghaiTime = &time n, ethservice := startEthService(t, genesis, blocks) - ethservice.Merger().ReachTTD() defer n.Close() api := NewConsensusAPI(ethservice) @@ -1162,7 +1160,6 @@ func TestNilWithdrawals(t *testing.T) { genesis.Config.ShanghaiTime = &time n, ethservice := startEthService(t, genesis, blocks) - ethservice.Merger().ReachTTD() defer n.Close() api := NewConsensusAPI(ethservice) @@ -1589,7 +1586,6 @@ func TestParentBeaconBlockRoot(t *testing.T) { genesis.Config.CancunTime = &time n, ethservice := startEthService(t, genesis, blocks) - ethservice.Merger().ReachTTD() defer n.Close() api := NewConsensusAPI(ethservice) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index ad664afb5bd1..420a8b147a7f 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -165,15 +165,14 @@ type Config struct { // Clique is allowed for now to live standalone, but ethash is forbidden and can // only exist on already merged networks. func CreateConsensusEngine(config *params.ChainConfig, db ethdb.Database) (consensus.Engine, error) { - // If proof-of-authority is requested, set it up + // Geth v1.14.0 dropped support for non-merged networks in any consensus + // mode. If such a network is requested, reject startup. + if !config.TerminalTotalDifficultyPassed { + return nil, errors.New("only PoS networks are supported, please transition old ones with Geth v1.13.x") + } + // Wrap previously supported consensus engines into their post-merge counterpart if config.Clique != nil { return beacon.New(clique.New(config.Clique, db)), nil } - // If defaulting to proof-of-work, enforce an already merged network since - // we cannot run PoW algorithms anymore, so we cannot even follow a chain - // not coordinated by a beacon node. - if !config.TerminalTotalDifficultyPassed { - return nil, errors.New("ethash is only supported as a historical component of already merged networks") - } return beacon.New(ethash.NewFaker()), nil } diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go deleted file mode 100644 index 126eaaea7fad..000000000000 --- a/eth/fetcher/block_fetcher.go +++ /dev/null @@ -1,939 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package fetcher contains the announcement based header, blocks or transaction synchronisation. -package fetcher - -import ( - "errors" - "math/rand" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/prque" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/trie" -) - -const ( - lightTimeout = time.Millisecond // Time allowance before an announced header is explicitly requested - arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block/transaction is explicitly requested - gatherSlack = 100 * time.Millisecond // Interval used to collate almost-expired announces with fetches - fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block/transaction -) - -const ( - maxUncleDist = 7 // Maximum allowed backward distance from the chain head - maxQueueDist = 32 // Maximum allowed distance from the chain head to queue - hashLimit = 256 // Maximum number of unique blocks or headers a peer may have announced - blockLimit = 64 // Maximum number of unique blocks a peer may have delivered -) - -var ( - blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/in", nil) - blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/announces/out", nil) - blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/drop", nil) - blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/dos", nil) - - blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/in", nil) - blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/broadcasts/out", nil) - blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/drop", nil) - blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/dos", nil) - - headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/headers", nil) - bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/bodies", nil) - - headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/in", nil) - headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/out", nil) - bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/in", nil) - bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/out", nil) -) - -var errTerminated = errors.New("terminated") - -// HeaderRetrievalFn is a callback type for retrieving a header from the local chain. -type HeaderRetrievalFn func(common.Hash) *types.Header - -// blockRetrievalFn is a callback type for retrieving a block from the local chain. -type blockRetrievalFn func(common.Hash) *types.Block - -// headerRequesterFn is a callback type for sending a header retrieval request. -type headerRequesterFn func(common.Hash, chan *eth.Response) (*eth.Request, error) - -// bodyRequesterFn is a callback type for sending a body retrieval request. -type bodyRequesterFn func([]common.Hash, chan *eth.Response) (*eth.Request, error) - -// headerVerifierFn is a callback type to verify a block's header for fast propagation. -type headerVerifierFn func(header *types.Header) error - -// blockBroadcasterFn is a callback type for broadcasting a block to connected peers. -type blockBroadcasterFn func(block *types.Block, propagate bool) - -// chainHeightFn is a callback type to retrieve the current chain height. -type chainHeightFn func() uint64 - -// headersInsertFn is a callback type to insert a batch of headers into the local chain. -type headersInsertFn func(headers []*types.Header) (int, error) - -// chainInsertFn is a callback type to insert a batch of blocks into the local chain. -type chainInsertFn func(types.Blocks) (int, error) - -// peerDropFn is a callback type for dropping a peer detected as malicious. -type peerDropFn func(id string) - -// blockAnnounce is the hash notification of the availability of a new block in the -// network. -type blockAnnounce struct { - hash common.Hash // Hash of the block being announced - number uint64 // Number of the block being announced (0 = unknown | old protocol) - header *types.Header // Header of the block partially reassembled (new protocol) - time time.Time // Timestamp of the announcement - - origin string // Identifier of the peer originating the notification - - fetchHeader headerRequesterFn // Fetcher function to retrieve the header of an announced block - fetchBodies bodyRequesterFn // Fetcher function to retrieve the body of an announced block -} - -// headerFilterTask represents a batch of headers needing fetcher filtering. -type headerFilterTask struct { - peer string // The source peer of block headers - headers []*types.Header // Collection of headers to filter - time time.Time // Arrival time of the headers -} - -// bodyFilterTask represents a batch of block bodies (transactions and uncles) -// needing fetcher filtering. -type bodyFilterTask struct { - peer string // The source peer of block bodies - transactions [][]*types.Transaction // Collection of transactions per block bodies - uncles [][]*types.Header // Collection of uncles per block bodies - time time.Time // Arrival time of the blocks' contents -} - -// blockOrHeaderInject represents a schedules import operation. -type blockOrHeaderInject struct { - origin string - - header *types.Header // Used for light mode fetcher which only cares about header. - block *types.Block // Used for normal mode fetcher which imports full block. -} - -// number returns the block number of the injected object. -func (inject *blockOrHeaderInject) number() uint64 { - if inject.header != nil { - return inject.header.Number.Uint64() - } - return inject.block.NumberU64() -} - -// number returns the block hash of the injected object. -func (inject *blockOrHeaderInject) hash() common.Hash { - if inject.header != nil { - return inject.header.Hash() - } - return inject.block.Hash() -} - -// BlockFetcher is responsible for accumulating block announcements from various peers -// and scheduling them for retrieval. -type BlockFetcher struct { - light bool // The indicator whether it's a light fetcher or normal one. - - // Various event channels - notify chan *blockAnnounce - inject chan *blockOrHeaderInject - - headerFilter chan chan *headerFilterTask - bodyFilter chan chan *bodyFilterTask - - done chan common.Hash - quit chan struct{} - - // Announce states - announces map[string]int // Per peer blockAnnounce counts to prevent memory exhaustion - announced map[common.Hash][]*blockAnnounce // Announced blocks, scheduled for fetching - fetching map[common.Hash]*blockAnnounce // Announced blocks, currently fetching - fetched map[common.Hash][]*blockAnnounce // Blocks with headers fetched, scheduled for body retrieval - completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing - - // Block cache - queue *prque.Prque[int64, *blockOrHeaderInject] // Queue containing the import operations (block number sorted) - queues map[string]int // Per peer block counts to prevent memory exhaustion - queued map[common.Hash]*blockOrHeaderInject // Set of already queued blocks (to dedup imports) - - // Callbacks - getHeader HeaderRetrievalFn // Retrieves a header from the local chain - getBlock blockRetrievalFn // Retrieves a block from the local chain - verifyHeader headerVerifierFn // Checks if a block's headers have a valid proof of work - broadcastBlock blockBroadcasterFn // Broadcasts a block to connected peers - chainHeight chainHeightFn // Retrieves the current chain's height - insertHeaders headersInsertFn // Injects a batch of headers into the chain - insertChain chainInsertFn // Injects a batch of blocks into the chain - dropPeer peerDropFn // Drops a peer for misbehaving - - // Testing hooks - announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the blockAnnounce list - queueChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a block from the import queue - fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch - completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62) - importedHook func(*types.Header, *types.Block) // Method to call upon successful header or block import (both eth/61 and eth/62) -} - -// NewBlockFetcher creates a block fetcher to retrieve blocks based on hash announcements. -func NewBlockFetcher(light bool, getHeader HeaderRetrievalFn, getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertHeaders headersInsertFn, insertChain chainInsertFn, dropPeer peerDropFn) *BlockFetcher { - return &BlockFetcher{ - light: light, - notify: make(chan *blockAnnounce), - inject: make(chan *blockOrHeaderInject), - headerFilter: make(chan chan *headerFilterTask), - bodyFilter: make(chan chan *bodyFilterTask), - done: make(chan common.Hash), - quit: make(chan struct{}), - announces: make(map[string]int), - announced: make(map[common.Hash][]*blockAnnounce), - fetching: make(map[common.Hash]*blockAnnounce), - fetched: make(map[common.Hash][]*blockAnnounce), - completing: make(map[common.Hash]*blockAnnounce), - queue: prque.New[int64, *blockOrHeaderInject](nil), - queues: make(map[string]int), - queued: make(map[common.Hash]*blockOrHeaderInject), - getHeader: getHeader, - getBlock: getBlock, - verifyHeader: verifyHeader, - broadcastBlock: broadcastBlock, - chainHeight: chainHeight, - insertHeaders: insertHeaders, - insertChain: insertChain, - dropPeer: dropPeer, - } -} - -// Start boots up the announcement based synchroniser, accepting and processing -// hash notifications and block fetches until termination requested. -func (f *BlockFetcher) Start() { - go f.loop() -} - -// Stop terminates the announcement based synchroniser, canceling all pending -// operations. -func (f *BlockFetcher) Stop() { - close(f.quit) -} - -// Notify announces the fetcher of the potential availability of a new block in -// the network. -func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, - headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error { - block := &blockAnnounce{ - hash: hash, - number: number, - time: time, - origin: peer, - fetchHeader: headerFetcher, - fetchBodies: bodyFetcher, - } - select { - case f.notify <- block: - return nil - case <-f.quit: - return errTerminated - } -} - -// Enqueue tries to fill gaps the fetcher's future import queue. -func (f *BlockFetcher) Enqueue(peer string, block *types.Block) error { - op := &blockOrHeaderInject{ - origin: peer, - block: block, - } - select { - case f.inject <- op: - return nil - case <-f.quit: - return errTerminated - } -} - -// FilterHeaders extracts all the headers that were explicitly requested by the fetcher, -// returning those that should be handled differently. -func (f *BlockFetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header { - log.Trace("Filtering headers", "peer", peer, "headers", len(headers)) - - // Send the filter channel to the fetcher - filter := make(chan *headerFilterTask) - - select { - case f.headerFilter <- filter: - case <-f.quit: - return nil - } - // Request the filtering of the header list - select { - case filter <- &headerFilterTask{peer: peer, headers: headers, time: time}: - case <-f.quit: - return nil - } - // Retrieve the headers remaining after filtering - select { - case task := <-filter: - return task.headers - case <-f.quit: - return nil - } -} - -// FilterBodies extracts all the block bodies that were explicitly requested by -// the fetcher, returning those that should be handled differently. -func (f *BlockFetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) { - log.Trace("Filtering bodies", "peer", peer, "txs", len(transactions), "uncles", len(uncles)) - - // Send the filter channel to the fetcher - filter := make(chan *bodyFilterTask) - - select { - case f.bodyFilter <- filter: - case <-f.quit: - return nil, nil - } - // Request the filtering of the body list - select { - case filter <- &bodyFilterTask{peer: peer, transactions: transactions, uncles: uncles, time: time}: - case <-f.quit: - return nil, nil - } - // Retrieve the bodies remaining after filtering - select { - case task := <-filter: - return task.transactions, task.uncles - case <-f.quit: - return nil, nil - } -} - -// Loop is the main fetcher loop, checking and processing various notification -// events. -func (f *BlockFetcher) loop() { - // Iterate the block fetching until a quit is requested - var ( - fetchTimer = time.NewTimer(0) - completeTimer = time.NewTimer(0) - ) - <-fetchTimer.C // clear out the channel - <-completeTimer.C - defer fetchTimer.Stop() - defer completeTimer.Stop() - - for { - // Clean up any expired block fetches - for hash, announce := range f.fetching { - if time.Since(announce.time) > fetchTimeout { - f.forgetHash(hash) - } - } - // Import any queued blocks that could potentially fit - height := f.chainHeight() - for !f.queue.Empty() { - op := f.queue.PopItem() - hash := op.hash() - if f.queueChangeHook != nil { - f.queueChangeHook(hash, false) - } - // If too high up the chain or phase, continue later - number := op.number() - if number > height+1 { - f.queue.Push(op, -int64(number)) - if f.queueChangeHook != nil { - f.queueChangeHook(hash, true) - } - break - } - // Otherwise if fresh and still unknown, try and import - if (number+maxUncleDist < height) || (f.light && f.getHeader(hash) != nil) || (!f.light && f.getBlock(hash) != nil) { - f.forgetBlock(hash) - continue - } - if f.light { - f.importHeaders(op.origin, op.header) - } else { - f.importBlocks(op.origin, op.block) - } - } - // Wait for an outside event to occur - select { - case <-f.quit: - // BlockFetcher terminating, abort all operations - return - - case notification := <-f.notify: - // A block was announced, make sure the peer isn't DOSing us - blockAnnounceInMeter.Mark(1) - - count := f.announces[notification.origin] + 1 - if count > hashLimit { - log.Debug("Peer exceeded outstanding announces", "peer", notification.origin, "limit", hashLimit) - blockAnnounceDOSMeter.Mark(1) - break - } - if notification.number == 0 { - break - } - // If we have a valid block number, check that it's potentially useful - if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { - log.Debug("Peer discarded announcement", "peer", notification.origin, "number", notification.number, "hash", notification.hash, "distance", dist) - blockAnnounceDropMeter.Mark(1) - break - } - // All is well, schedule the announce if block's not yet downloading - if _, ok := f.fetching[notification.hash]; ok { - break - } - if _, ok := f.completing[notification.hash]; ok { - break - } - f.announces[notification.origin] = count - f.announced[notification.hash] = append(f.announced[notification.hash], notification) - if f.announceChangeHook != nil && len(f.announced[notification.hash]) == 1 { - f.announceChangeHook(notification.hash, true) - } - if len(f.announced) == 1 { - f.rescheduleFetch(fetchTimer) - } - - case op := <-f.inject: - // A direct block insertion was requested, try and fill any pending gaps - blockBroadcastInMeter.Mark(1) - - // Now only direct block injection is allowed, drop the header injection - // here silently if we receive. - if f.light { - continue - } - f.enqueue(op.origin, nil, op.block) - - case hash := <-f.done: - // A pending import finished, remove all traces of the notification - f.forgetHash(hash) - f.forgetBlock(hash) - - case <-fetchTimer.C: - // At least one block's timer ran out, check for needing retrieval - request := make(map[string][]common.Hash) - - for hash, announces := range f.announced { - // In current LES protocol(les2/les3), only header announce is - // available, no need to wait too much time for header broadcast. - timeout := arriveTimeout - gatherSlack - if f.light { - timeout = 0 - } - if time.Since(announces[0].time) > timeout { - // Pick a random peer to retrieve from, reset all others - announce := announces[rand.Intn(len(announces))] - f.forgetHash(hash) - - // If the block still didn't arrive, queue for fetching - if (f.light && f.getHeader(hash) == nil) || (!f.light && f.getBlock(hash) == nil) { - request[announce.origin] = append(request[announce.origin], hash) - f.fetching[hash] = announce - } - } - } - // Send out all block header requests - for peer, hashes := range request { - log.Trace("Fetching scheduled headers", "peer", peer, "list", hashes) - - // Create a closure of the fetch and schedule in on a new thread - fetchHeader, hashes := f.fetching[hashes[0]].fetchHeader, hashes - go func(peer string) { - if f.fetchingHook != nil { - f.fetchingHook(hashes) - } - for _, hash := range hashes { - headerFetchMeter.Mark(1) - go func(hash common.Hash) { - resCh := make(chan *eth.Response) - - req, err := fetchHeader(hash, resCh) - if err != nil { - return // Legacy code, yolo - } - defer req.Close() - - timeout := time.NewTimer(2 * fetchTimeout) // 2x leeway before dropping the peer - defer timeout.Stop() - - select { - case res := <-resCh: - res.Done <- nil - f.FilterHeaders(peer, *res.Res.(*eth.BlockHeadersRequest), time.Now()) - - case <-timeout.C: - // The peer didn't respond in time. The request - // was already rescheduled at this point, we were - // waiting for a catchup. With an unresponsive - // peer however, it's a protocol violation. - f.dropPeer(peer) - } - }(hash) - } - }(peer) - } - // Schedule the next fetch if blocks are still pending - f.rescheduleFetch(fetchTimer) - - case <-completeTimer.C: - // At least one header's timer ran out, retrieve everything - request := make(map[string][]common.Hash) - - for hash, announces := range f.fetched { - // Pick a random peer to retrieve from, reset all others - announce := announces[rand.Intn(len(announces))] - f.forgetHash(hash) - - // If the block still didn't arrive, queue for completion - if f.getBlock(hash) == nil { - request[announce.origin] = append(request[announce.origin], hash) - f.completing[hash] = announce - } - } - // Send out all block body requests - for peer, hashes := range request { - log.Trace("Fetching scheduled bodies", "peer", peer, "list", hashes) - - // Create a closure of the fetch and schedule in on a new thread - if f.completingHook != nil { - f.completingHook(hashes) - } - fetchBodies := f.completing[hashes[0]].fetchBodies - bodyFetchMeter.Mark(int64(len(hashes))) - - go func(peer string, hashes []common.Hash) { - resCh := make(chan *eth.Response) - - req, err := fetchBodies(hashes, resCh) - if err != nil { - return // Legacy code, yolo - } - defer req.Close() - - timeout := time.NewTimer(2 * fetchTimeout) // 2x leeway before dropping the peer - defer timeout.Stop() - - select { - case res := <-resCh: - res.Done <- nil - // Ignoring withdrawals here, since the block fetcher is not used post-merge. - txs, uncles, _ := res.Res.(*eth.BlockBodiesResponse).Unpack() - f.FilterBodies(peer, txs, uncles, time.Now()) - - case <-timeout.C: - // The peer didn't respond in time. The request - // was already rescheduled at this point, we were - // waiting for a catchup. With an unresponsive - // peer however, it's a protocol violation. - f.dropPeer(peer) - } - }(peer, hashes) - } - // Schedule the next fetch if blocks are still pending - f.rescheduleComplete(completeTimer) - - case filter := <-f.headerFilter: - // Headers arrived from a remote peer. Extract those that were explicitly - // requested by the fetcher, and return everything else so it's delivered - // to other parts of the system. - var task *headerFilterTask - select { - case task = <-filter: - case <-f.quit: - return - } - headerFilterInMeter.Mark(int64(len(task.headers))) - - // Split the batch of headers into unknown ones (to return to the caller), - // known incomplete ones (requiring body retrievals) and completed blocks. - unknown, incomplete, complete, lightHeaders := []*types.Header{}, []*blockAnnounce{}, []*types.Block{}, []*blockAnnounce{} - for _, header := range task.headers { - hash := header.Hash() - - // Filter fetcher-requested headers from other synchronisation algorithms - if announce := f.fetching[hash]; announce != nil && announce.origin == task.peer && f.fetched[hash] == nil && f.completing[hash] == nil && f.queued[hash] == nil { - // If the delivered header does not match the promised number, drop the announcer - if header.Number.Uint64() != announce.number { - log.Trace("Invalid block number fetched", "peer", announce.origin, "hash", header.Hash(), "announced", announce.number, "provided", header.Number) - f.dropPeer(announce.origin) - f.forgetHash(hash) - continue - } - // Collect all headers only if we are running in light - // mode and the headers are not imported by other means. - if f.light { - if f.getHeader(hash) == nil { - announce.header = header - lightHeaders = append(lightHeaders, announce) - } - f.forgetHash(hash) - continue - } - // Only keep if not imported by other means - if f.getBlock(hash) == nil { - announce.header = header - announce.time = task.time - - // If the block is empty (header only), short circuit into the final import queue - if header.TxHash == types.EmptyTxsHash && header.UncleHash == types.EmptyUncleHash { - log.Trace("Block empty, skipping body retrieval", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) - - block := types.NewBlockWithHeader(header) - block.ReceivedAt = task.time - - complete = append(complete, block) - f.completing[hash] = announce - continue - } - // Otherwise add to the list of blocks needing completion - incomplete = append(incomplete, announce) - } else { - log.Trace("Block already imported, discarding header", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) - f.forgetHash(hash) - } - } else { - // BlockFetcher doesn't know about it, add to the return list - unknown = append(unknown, header) - } - } - headerFilterOutMeter.Mark(int64(len(unknown))) - select { - case filter <- &headerFilterTask{headers: unknown, time: task.time}: - case <-f.quit: - return - } - // Schedule the retrieved headers for body completion - for _, announce := range incomplete { - hash := announce.header.Hash() - if _, ok := f.completing[hash]; ok { - continue - } - f.fetched[hash] = append(f.fetched[hash], announce) - if len(f.fetched) == 1 { - f.rescheduleComplete(completeTimer) - } - } - // Schedule the header for light fetcher import - for _, announce := range lightHeaders { - f.enqueue(announce.origin, announce.header, nil) - } - // Schedule the header-only blocks for import - for _, block := range complete { - if announce := f.completing[block.Hash()]; announce != nil { - f.enqueue(announce.origin, nil, block) - } - } - - case filter := <-f.bodyFilter: - // Block bodies arrived, extract any explicitly requested blocks, return the rest - var task *bodyFilterTask - select { - case task = <-filter: - case <-f.quit: - return - } - bodyFilterInMeter.Mark(int64(len(task.transactions))) - blocks := []*types.Block{} - // abort early if there's nothing explicitly requested - if len(f.completing) > 0 { - for i := 0; i < len(task.transactions) && i < len(task.uncles); i++ { - // Match up a body to any possible completion request - var ( - matched = false - uncleHash common.Hash // calculated lazily and reused - txnHash common.Hash // calculated lazily and reused - ) - for hash, announce := range f.completing { - if f.queued[hash] != nil || announce.origin != task.peer { - continue - } - if uncleHash == (common.Hash{}) { - uncleHash = types.CalcUncleHash(task.uncles[i]) - } - if uncleHash != announce.header.UncleHash { - continue - } - if txnHash == (common.Hash{}) { - txnHash = types.DeriveSha(types.Transactions(task.transactions[i]), trie.NewStackTrie(nil)) - } - if txnHash != announce.header.TxHash { - continue - } - // Mark the body matched, reassemble if still unknown - matched = true - if f.getBlock(hash) == nil { - block := types.NewBlockWithHeader(announce.header).WithBody(task.transactions[i], task.uncles[i]) - block.ReceivedAt = task.time - blocks = append(blocks, block) - } else { - f.forgetHash(hash) - } - } - if matched { - task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) - task.uncles = append(task.uncles[:i], task.uncles[i+1:]...) - i-- - continue - } - } - } - bodyFilterOutMeter.Mark(int64(len(task.transactions))) - select { - case filter <- task: - case <-f.quit: - return - } - // Schedule the retrieved blocks for ordered import - for _, block := range blocks { - if announce := f.completing[block.Hash()]; announce != nil { - f.enqueue(announce.origin, nil, block) - } - } - } - } -} - -// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. -func (f *BlockFetcher) rescheduleFetch(fetch *time.Timer) { - // Short circuit if no blocks are announced - if len(f.announced) == 0 { - return - } - // Schedule announcement retrieval quickly for light mode - // since server won't send any headers to client. - if f.light { - fetch.Reset(lightTimeout) - return - } - // Otherwise find the earliest expiring announcement - earliest := time.Now() - for _, announces := range f.announced { - if earliest.After(announces[0].time) { - earliest = announces[0].time - } - } - fetch.Reset(arriveTimeout - time.Since(earliest)) -} - -// rescheduleComplete resets the specified completion timer to the next fetch timeout. -func (f *BlockFetcher) rescheduleComplete(complete *time.Timer) { - // Short circuit if no headers are fetched - if len(f.fetched) == 0 { - return - } - // Otherwise find the earliest expiring announcement - earliest := time.Now() - for _, announces := range f.fetched { - if earliest.After(announces[0].time) { - earliest = announces[0].time - } - } - complete.Reset(gatherSlack - time.Since(earliest)) -} - -// enqueue schedules a new header or block import operation, if the component -// to be imported has not yet been seen. -func (f *BlockFetcher) enqueue(peer string, header *types.Header, block *types.Block) { - var ( - hash common.Hash - number uint64 - ) - if header != nil { - hash, number = header.Hash(), header.Number.Uint64() - } else { - hash, number = block.Hash(), block.NumberU64() - } - // Ensure the peer isn't DOSing us - count := f.queues[peer] + 1 - if count > blockLimit { - log.Debug("Discarded delivered header or block, exceeded allowance", "peer", peer, "number", number, "hash", hash, "limit", blockLimit) - blockBroadcastDOSMeter.Mark(1) - f.forgetHash(hash) - return - } - // Discard any past or too distant blocks - if dist := int64(number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { - log.Debug("Discarded delivered header or block, too far away", "peer", peer, "number", number, "hash", hash, "distance", dist) - blockBroadcastDropMeter.Mark(1) - f.forgetHash(hash) - return - } - // Schedule the block for future importing - if _, ok := f.queued[hash]; !ok { - op := &blockOrHeaderInject{origin: peer} - if header != nil { - op.header = header - } else { - op.block = block - } - f.queues[peer] = count - f.queued[hash] = op - f.queue.Push(op, -int64(number)) - if f.queueChangeHook != nil { - f.queueChangeHook(hash, true) - } - log.Debug("Queued delivered header or block", "peer", peer, "number", number, "hash", hash, "queued", f.queue.Size()) - } -} - -// importHeaders spawns a new goroutine to run a header insertion into the chain. -// If the header's number is at the same height as the current import phase, it -// updates the phase states accordingly. -func (f *BlockFetcher) importHeaders(peer string, header *types.Header) { - hash := header.Hash() - log.Debug("Importing propagated header", "peer", peer, "number", header.Number, "hash", hash) - - go func() { - defer func() { f.done <- hash }() - // If the parent's unknown, abort insertion - parent := f.getHeader(header.ParentHash) - if parent == nil { - log.Debug("Unknown parent of propagated header", "peer", peer, "number", header.Number, "hash", hash, "parent", header.ParentHash) - return - } - // Validate the header and if something went wrong, drop the peer - if err := f.verifyHeader(header); err != nil && err != consensus.ErrFutureBlock { - log.Debug("Propagated header verification failed", "peer", peer, "number", header.Number, "hash", hash, "err", err) - f.dropPeer(peer) - return - } - // Run the actual import and log any issues - if _, err := f.insertHeaders([]*types.Header{header}); err != nil { - log.Debug("Propagated header import failed", "peer", peer, "number", header.Number, "hash", hash, "err", err) - return - } - // Invoke the testing hook if needed - if f.importedHook != nil { - f.importedHook(header, nil) - } - }() -} - -// importBlocks spawns a new goroutine to run a block insertion into the chain. If the -// block's number is at the same height as the current import phase, it updates -// the phase states accordingly. -func (f *BlockFetcher) importBlocks(peer string, block *types.Block) { - hash := block.Hash() - - // Run the import on a new thread - log.Debug("Importing propagated block", "peer", peer, "number", block.Number(), "hash", hash) - go func() { - defer func() { f.done <- hash }() - - // If the parent's unknown, abort insertion - parent := f.getBlock(block.ParentHash()) - if parent == nil { - log.Debug("Unknown parent of propagated block", "peer", peer, "number", block.Number(), "hash", hash, "parent", block.ParentHash()) - return - } - // Quickly validate the header and propagate the block if it passes - switch err := f.verifyHeader(block.Header()); err { - case nil: - // All ok, quickly propagate to our peers - blockBroadcastOutTimer.UpdateSince(block.ReceivedAt) - go f.broadcastBlock(block, true) - - case consensus.ErrFutureBlock: - // Weird future block, don't fail, but neither propagate - - default: - // Something went very wrong, drop the peer - log.Debug("Propagated block verification failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err) - f.dropPeer(peer) - return - } - // Run the actual import and log any issues - if _, err := f.insertChain(types.Blocks{block}); err != nil { - log.Debug("Propagated block import failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err) - return - } - // If import succeeded, broadcast the block - blockAnnounceOutTimer.UpdateSince(block.ReceivedAt) - go f.broadcastBlock(block, false) - - // Invoke the testing hook if needed - if f.importedHook != nil { - f.importedHook(nil, block) - } - }() -} - -// forgetHash removes all traces of a block announcement from the fetcher's -// internal state. -func (f *BlockFetcher) forgetHash(hash common.Hash) { - // Remove all pending announces and decrement DOS counters - if announceMap, ok := f.announced[hash]; ok { - for _, announce := range announceMap { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - } - delete(f.announced, hash) - if f.announceChangeHook != nil { - f.announceChangeHook(hash, false) - } - } - // Remove any pending fetches and decrement the DOS counters - if announce := f.fetching[hash]; announce != nil { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - delete(f.fetching, hash) - } - - // Remove any pending completion requests and decrement the DOS counters - for _, announce := range f.fetched[hash] { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - } - delete(f.fetched, hash) - - // Remove any pending completions and decrement the DOS counters - if announce := f.completing[hash]; announce != nil { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - delete(f.completing, hash) - } -} - -// forgetBlock removes all traces of a queued block from the fetcher's internal -// state. -func (f *BlockFetcher) forgetBlock(hash common.Hash) { - if insert := f.queued[hash]; insert != nil { - f.queues[insert.origin]-- - if f.queues[insert.origin] == 0 { - delete(f.queues, insert.origin) - } - delete(f.queued, hash) - } -} diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go deleted file mode 100644 index cb7cbaf79edc..000000000000 --- a/eth/fetcher/block_fetcher_test.go +++ /dev/null @@ -1,949 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package fetcher - -import ( - "errors" - "math/big" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/triedb" -) - -var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - gspec = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: types.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - genesis = gspec.MustCommit(testdb, triedb.NewDatabase(testdb, triedb.HashDefaults)) - unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) -) - -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { - blocks, _ := core.GenerateChain(gspec.Config, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { - block.SetCoinbase(common.Address{seed}) - - // If the block number is multiple of 3, send a bonus transaction to the miner - if parent == genesis && i%3 == 0 { - signer := types.MakeSigner(params.TestChainConfig, block.Number(), block.Timestamp()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) - if err != nil { - panic(err) - } - block.AddTx(tx) - } - // If the block number is a multiple of 5, add a bonus uncle to the block - if i > 0 && i%5 == 0 { - block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 2).Hash(), Number: big.NewInt(int64(i - 1))}) - } - }) - hashes := make([]common.Hash, n+1) - hashes[len(hashes)-1] = parent.Hash() - blockm := make(map[common.Hash]*types.Block, n+1) - blockm[parent.Hash()] = parent - for i, b := range blocks { - hashes[len(hashes)-i-2] = b.Hash() - blockm[b.Hash()] = b - } - return hashes, blockm -} - -// fetcherTester is a test simulator for mocking out local block chain. -type fetcherTester struct { - fetcher *BlockFetcher - - hashes []common.Hash // Hash chain belonging to the tester - headers map[common.Hash]*types.Header // Headers belonging to the tester - blocks map[common.Hash]*types.Block // Blocks belonging to the tester - drops map[string]bool // Map of peers dropped by the fetcher - - lock sync.RWMutex -} - -// newTester creates a new fetcher test mocker. -func newTester(light bool) *fetcherTester { - tester := &fetcherTester{ - hashes: []common.Hash{genesis.Hash()}, - headers: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, - blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, - drops: make(map[string]bool), - } - tester.fetcher = NewBlockFetcher(light, tester.getHeader, tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertHeaders, tester.insertChain, tester.dropPeer) - tester.fetcher.Start() - - return tester -} - -// getHeader retrieves a header from the tester's block chain. -func (f *fetcherTester) getHeader(hash common.Hash) *types.Header { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.headers[hash] -} - -// getBlock retrieves a block from the tester's block chain. -func (f *fetcherTester) getBlock(hash common.Hash) *types.Block { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.blocks[hash] -} - -// verifyHeader is a nop placeholder for the block header verification. -func (f *fetcherTester) verifyHeader(header *types.Header) error { - return nil -} - -// broadcastBlock is a nop placeholder for the block broadcasting. -func (f *fetcherTester) broadcastBlock(block *types.Block, propagate bool) { -} - -// chainHeight retrieves the current height (block number) of the chain. -func (f *fetcherTester) chainHeight() uint64 { - f.lock.RLock() - defer f.lock.RUnlock() - - if f.fetcher.light { - return f.headers[f.hashes[len(f.hashes)-1]].Number.Uint64() - } - return f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() -} - -// insertChain injects a new headers into the simulated chain. -func (f *fetcherTester) insertHeaders(headers []*types.Header) (int, error) { - f.lock.Lock() - defer f.lock.Unlock() - - for i, header := range headers { - // Make sure the parent in known - if _, ok := f.headers[header.ParentHash]; !ok { - return i, errors.New("unknown parent") - } - // Discard any new blocks if the same height already exists - if header.Number.Uint64() <= f.headers[f.hashes[len(f.hashes)-1]].Number.Uint64() { - return i, nil - } - // Otherwise build our current chain - f.hashes = append(f.hashes, header.Hash()) - f.headers[header.Hash()] = header - } - return 0, nil -} - -// insertChain injects a new blocks into the simulated chain. -func (f *fetcherTester) insertChain(blocks types.Blocks) (int, error) { - f.lock.Lock() - defer f.lock.Unlock() - - for i, block := range blocks { - // Make sure the parent in known - if _, ok := f.blocks[block.ParentHash()]; !ok { - return i, errors.New("unknown parent") - } - // Discard any new blocks if the same height already exists - if block.NumberU64() <= f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() { - return i, nil - } - // Otherwise build our current chain - f.hashes = append(f.hashes, block.Hash()) - f.blocks[block.Hash()] = block - } - return 0, nil -} - -// dropPeer is an emulator for the peer removal, simply accumulating the various -// peers dropped by the fetcher. -func (f *fetcherTester) dropPeer(peer string) { - f.lock.Lock() - defer f.lock.Unlock() - - f.drops[peer] = true -} - -// makeHeaderFetcher retrieves a block header fetcher associated with a simulated peer. -func (f *fetcherTester) makeHeaderFetcher(peer string, blocks map[common.Hash]*types.Block, drift time.Duration) headerRequesterFn { - closure := make(map[common.Hash]*types.Block) - for hash, block := range blocks { - closure[hash] = block - } - // Create a function that return a header from the closure - return func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - // Gather the blocks to return - headers := make([]*types.Header, 0, 1) - if block, ok := closure[hash]; ok { - headers = append(headers, block.Header()) - } - // Return on a new thread - req := ð.Request{ - Peer: peer, - } - res := ð.Response{ - Req: req, - Res: (*eth.BlockHeadersRequest)(&headers), - Time: drift, - Done: make(chan error, 1), // Ignore the returned status - } - go func() { - sink <- res - }() - return req, nil - } -} - -// makeBodyFetcher retrieves a block body fetcher associated with a simulated peer. -func (f *fetcherTester) makeBodyFetcher(peer string, blocks map[common.Hash]*types.Block, drift time.Duration) bodyRequesterFn { - closure := make(map[common.Hash]*types.Block) - for hash, block := range blocks { - closure[hash] = block - } - // Create a function that returns blocks from the closure - return func(hashes []common.Hash, sink chan *eth.Response) (*eth.Request, error) { - // Gather the block bodies to return - transactions := make([][]*types.Transaction, 0, len(hashes)) - uncles := make([][]*types.Header, 0, len(hashes)) - - for _, hash := range hashes { - if block, ok := closure[hash]; ok { - transactions = append(transactions, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } - } - // Return on a new thread - bodies := make([]*eth.BlockBody, len(transactions)) - for i, txs := range transactions { - bodies[i] = ð.BlockBody{ - Transactions: txs, - Uncles: uncles[i], - } - } - req := ð.Request{ - Peer: peer, - } - res := ð.Response{ - Req: req, - Res: (*eth.BlockBodiesResponse)(&bodies), - Time: drift, - Done: make(chan error, 1), // Ignore the returned status - } - go func() { - sink <- res - }() - return req, nil - } -} - -// verifyFetchingEvent verifies that one single event arrive on a fetching channel. -func verifyFetchingEvent(t *testing.T, fetching chan []common.Hash, arrive bool) { - t.Helper() - - if arrive { - select { - case <-fetching: - case <-time.After(time.Second): - t.Fatalf("fetching timeout") - } - } else { - select { - case <-fetching: - t.Fatalf("fetching invoked") - case <-time.After(10 * time.Millisecond): - } - } -} - -// verifyCompletingEvent verifies that one single event arrive on an completing channel. -func verifyCompletingEvent(t *testing.T, completing chan []common.Hash, arrive bool) { - t.Helper() - - if arrive { - select { - case <-completing: - case <-time.After(time.Second): - t.Fatalf("completing timeout") - } - } else { - select { - case <-completing: - t.Fatalf("completing invoked") - case <-time.After(10 * time.Millisecond): - } - } -} - -// verifyImportEvent verifies that one single event arrive on an import channel. -func verifyImportEvent(t *testing.T, imported chan interface{}, arrive bool) { - t.Helper() - - if arrive { - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("import timeout") - } - } else { - select { - case <-imported: - t.Fatalf("import invoked") - case <-time.After(20 * time.Millisecond): - } - } -} - -// verifyImportCount verifies that exactly count number of events arrive on an -// import hook channel. -func verifyImportCount(t *testing.T, imported chan interface{}, count int) { - t.Helper() - - for i := 0; i < count; i++ { - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("block %d: import timeout", i+1) - } - } - verifyImportDone(t, imported) -} - -// verifyImportDone verifies that no more events are arriving on an import channel. -func verifyImportDone(t *testing.T, imported chan interface{}) { - t.Helper() - - select { - case <-imported: - t.Fatalf("extra block imported") - case <-time.After(50 * time.Millisecond): - } -} - -// verifyChainHeight verifies the chain height is as expected. -func verifyChainHeight(t *testing.T, fetcher *fetcherTester, height uint64) { - t.Helper() - - if fetcher.chainHeight() != height { - t.Fatalf("chain height mismatch, got %d, want %d", fetcher.chainHeight(), height) - } -} - -// Tests that a fetcher accepts block/header announcements and initiates retrievals -// for them, successfully importing into the local chain. -func TestFullSequentialAnnouncements(t *testing.T) { testSequentialAnnouncements(t, false) } -func TestLightSequentialAnnouncements(t *testing.T) { testSequentialAnnouncements(t, true) } - -func testSequentialAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - tester := newTester(light) - defer tester.fetcher.Stop() - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks until all are imported - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that if blocks are announced by multiple peers (or even the same buggy -// peer), they will only get downloaded at most once. -func TestFullConcurrentAnnouncements(t *testing.T) { testConcurrentAnnouncements(t, false) } -func TestLightConcurrentAnnouncements(t *testing.T) { testConcurrentAnnouncements(t, true) } - -func testConcurrentAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - // Assemble a tester with a built in counter for the requests - tester := newTester(light) - firstHeaderFetcher := tester.makeHeaderFetcher("first", blocks, -gatherSlack) - firstBodyFetcher := tester.makeBodyFetcher("first", blocks, 0) - secondHeaderFetcher := tester.makeHeaderFetcher("second", blocks, -gatherSlack) - secondBodyFetcher := tester.makeBodyFetcher("second", blocks, 0) - - var counter atomic.Uint32 - firstHeaderWrapper := func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - counter.Add(1) - return firstHeaderFetcher(hash, sink) - } - secondHeaderWrapper := func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - counter.Add(1) - return secondHeaderFetcher(hash, sink) - } - // Iteratively announce blocks until all are imported - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), firstHeaderWrapper, firstBodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), secondHeaderWrapper, secondBodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), secondHeaderWrapper, secondBodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) - - // Make sure no blocks were retrieved twice - if c := int(counter.Load()); c != targetBlocks { - t.Fatalf("retrieval count mismatch: have %v, want %v", c, targetBlocks) - } - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that announcements arriving while a previous is being fetched still -// results in a valid import. -func TestFullOverlappingAnnouncements(t *testing.T) { testOverlappingAnnouncements(t, false) } -func TestLightOverlappingAnnouncements(t *testing.T) { testOverlappingAnnouncements(t, true) } - -func testOverlappingAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, but overlap them continuously - overlap := 16 - imported := make(chan interface{}, len(hashes)-1) - for i := 0; i < overlap; i++ { - imported <- nil - } - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("block %d: import timeout", len(hashes)-i) - } - } - // Wait for all the imports to complete and check count - verifyImportCount(t, imported, overlap) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that announces already being retrieved will not be duplicated. -func TestFullPendingDeduplication(t *testing.T) { testPendingDeduplication(t, false) } -func TestLightPendingDeduplication(t *testing.T) { testPendingDeduplication(t, true) } - -func testPendingDeduplication(t *testing.T, light bool) { - // Create a hash and corresponding block - hashes, blocks := makeChain(1, 0, genesis) - - // Assemble a tester with a built in counter and delayed fetcher - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("repeater", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("repeater", blocks, 0) - - delay := 50 * time.Millisecond - var counter atomic.Uint32 - headerWrapper := func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - counter.Add(1) - - // Simulate a long running fetch - resink := make(chan *eth.Response) - req, err := headerFetcher(hash, resink) - if err == nil { - go func() { - res := <-resink - time.Sleep(delay) - sink <- res - }() - } - return req, err - } - checkNonExist := func() bool { - return tester.getBlock(hashes[0]) == nil - } - if light { - checkNonExist = func() bool { - return tester.getHeader(hashes[0]) == nil - } - } - // Announce the same block many times until it's fetched (wait for any pending ops) - for checkNonExist() { - tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher) - time.Sleep(time.Millisecond) - } - time.Sleep(delay) - - // Check that all blocks were imported and none fetched twice - if c := counter.Load(); c != 1 { - t.Fatalf("retrieval count mismatch: have %v, want %v", c, 1) - } - verifyChainHeight(t, tester, 1) -} - -// Tests that announcements retrieved in a random order are cached and eventually -// imported when all the gaps are filled in. -func TestFullRandomArrivalImport(t *testing.T) { testRandomArrivalImport(t, false) } -func TestLightRandomArrivalImport(t *testing.T) { testRandomArrivalImport(t, true) } - -func testRandomArrivalImport(t *testing.T, light bool) { - // Create a chain of blocks to import, and choose one to delay - targetBlocks := maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - skip := targetBlocks / 2 - - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, skipping one entry - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 1; i >= 0; i-- { - if i != skip { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - time.Sleep(time.Millisecond) - } - } - // Finally announce the skipped entry and check full import - tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - verifyImportCount(t, imported, len(hashes)-1) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that direct block enqueues (due to block propagation vs. hash announce) -// are correctly schedule, filling and import queue gaps. -func TestQueueGapFill(t *testing.T) { - // Create a chain of blocks to import, and choose one to not announce at all - targetBlocks := maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - skip := targetBlocks / 2 - - tester := newTester(false) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, skipping one entry - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - - for i := len(hashes) - 1; i >= 0; i-- { - if i != skip { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - time.Sleep(time.Millisecond) - } - } - // Fill the missing block directly as if propagated - tester.fetcher.Enqueue("valid", blocks[hashes[skip]]) - verifyImportCount(t, imported, len(hashes)-1) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that blocks arriving from various sources (multiple propagations, hash -// announces, etc) do not get scheduled for import multiple times. -func TestImportDeduplication(t *testing.T) { - // Create two blocks to import (one for duplication, the other for stalling) - hashes, blocks := makeChain(2, 0, genesis) - - // Create the tester and wrap the importer with a counter - tester := newTester(false) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - var counter atomic.Uint32 - tester.fetcher.insertChain = func(blocks types.Blocks) (int, error) { - counter.Add(uint32(len(blocks))) - return tester.insertChain(blocks) - } - // Instrument the fetching and imported events - fetching := make(chan []common.Hash) - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - - // Announce the duplicating block, wait for retrieval, and also propagate directly - tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - <-fetching - - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - - // Fill the missing block directly as if propagated, and check import uniqueness - tester.fetcher.Enqueue("valid", blocks[hashes[1]]) - verifyImportCount(t, imported, 2) - - if c := counter.Load(); c != 2 { - t.Fatalf("import invocation count mismatch: have %v, want %v", c, 2) - } -} - -// Tests that blocks with numbers much lower or higher than out current head get -// discarded to prevent wasting resources on useless blocks from faulty peers. -func TestDistantPropagationDiscarding(t *testing.T) { - // Create a long chain to import and define the discard boundaries - hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) - head := hashes[len(hashes)/2] - - low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1 - - // Create a tester and simulate a head block being the middle of the above chain - tester := newTester(false) - - tester.lock.Lock() - tester.hashes = []common.Hash{head} - tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} - tester.lock.Unlock() - - // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Enqueue("lower", blocks[hashes[low]]) - time.Sleep(10 * time.Millisecond) - if !tester.fetcher.queue.Empty() { - t.Fatalf("fetcher queued stale block") - } - // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Enqueue("higher", blocks[hashes[high]]) - time.Sleep(10 * time.Millisecond) - if !tester.fetcher.queue.Empty() { - t.Fatalf("fetcher queued future block") - } -} - -// Tests that announcements with numbers much lower or higher than out current -// head get discarded to prevent wasting resources on useless blocks from faulty -// peers. -func TestFullDistantAnnouncementDiscarding(t *testing.T) { testDistantAnnouncementDiscarding(t, false) } -func TestLightDistantAnnouncementDiscarding(t *testing.T) { testDistantAnnouncementDiscarding(t, true) } - -func testDistantAnnouncementDiscarding(t *testing.T, light bool) { - // Create a long chain to import and define the discard boundaries - hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) - head := hashes[len(hashes)/2] - - low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1 - - // Create a tester and simulate a head block being the middle of the above chain - tester := newTester(light) - - tester.lock.Lock() - tester.hashes = []common.Hash{head} - tester.headers = map[common.Hash]*types.Header{head: blocks[head].Header()} - tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} - tester.lock.Unlock() - - headerFetcher := tester.makeHeaderFetcher("lower", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("lower", blocks, 0) - - fetching := make(chan struct{}, 2) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- struct{}{} } - - // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-time.After(50 * time.Millisecond): - case <-fetching: - t.Fatalf("fetcher requested stale header") - } - // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-time.After(50 * time.Millisecond): - case <-fetching: - t.Fatalf("fetcher requested future header") - } -} - -// Tests that peers announcing blocks with invalid numbers (i.e. not matching -// the headers provided afterwards) get dropped as malicious. -func TestFullInvalidNumberAnnouncement(t *testing.T) { testInvalidNumberAnnouncement(t, false) } -func TestLightInvalidNumberAnnouncement(t *testing.T) { testInvalidNumberAnnouncement(t, true) } - -func testInvalidNumberAnnouncement(t *testing.T, light bool) { - // Create a single block to import and check numbers against - hashes, blocks := makeChain(1, 0, genesis) - - tester := newTester(light) - badHeaderFetcher := tester.makeHeaderFetcher("bad", blocks, -gatherSlack) - badBodyFetcher := tester.makeBodyFetcher("bad", blocks, 0) - - imported := make(chan interface{}) - announced := make(chan interface{}, 2) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - // Announce a block with a bad number, check for immediate drop - tester.fetcher.announceChangeHook = func(hash common.Hash, b bool) { - announced <- nil - } - tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), badHeaderFetcher, badBodyFetcher) - verifyAnnounce := func() { - for i := 0; i < 2; i++ { - select { - case <-announced: - continue - case <-time.After(1 * time.Second): - t.Fatal("announce timeout") - return - } - } - } - verifyAnnounce() - verifyImportEvent(t, imported, false) - tester.lock.RLock() - dropped := tester.drops["bad"] - tester.lock.RUnlock() - - if !dropped { - t.Fatalf("peer with invalid numbered announcement not dropped") - } - goodHeaderFetcher := tester.makeHeaderFetcher("good", blocks, -gatherSlack) - goodBodyFetcher := tester.makeBodyFetcher("good", blocks, 0) - // Make sure a good announcement passes without a drop - tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), goodHeaderFetcher, goodBodyFetcher) - verifyAnnounce() - verifyImportEvent(t, imported, true) - - tester.lock.RLock() - dropped = tester.drops["good"] - tester.lock.RUnlock() - - if dropped { - t.Fatalf("peer with valid numbered announcement dropped") - } - verifyImportDone(t, imported) -} - -// Tests that if a block is empty (i.e. header only), no body request should be -// made, and instead the header should be assembled into a whole block in itself. -func TestEmptyBlockShortCircuit(t *testing.T) { - // Create a chain of blocks to import - hashes, blocks := makeChain(32, 0, genesis) - - tester := newTester(false) - defer tester.fetcher.Stop() - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Add a monitoring hook for all internal events - fetching := make(chan []common.Hash) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } - - completing := make(chan []common.Hash) - tester.fetcher.completingHook = func(hashes []common.Hash) { completing <- hashes } - - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - // Iteratively announce blocks until all are imported - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - - // All announces should fetch the header - verifyFetchingEvent(t, fetching, true) - - // Only blocks with data contents should request bodies - verifyCompletingEvent(t, completing, len(blocks[hashes[i]].Transactions()) > 0 || len(blocks[hashes[i]].Uncles()) > 0) - - // Irrelevant of the construct, import should succeed - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} - -// Tests that a peer is unable to use unbounded memory with sending infinite -// block announcements to a node, but that even in the face of such an attack, -// the fetcher remains operational. -func TestHashMemoryExhaustionAttack(t *testing.T) { - // Create a tester with instrumented import hooks - tester := newTester(false) - - imported, announces := make(chan interface{}), atomic.Int32{} - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - tester.fetcher.announceChangeHook = func(hash common.Hash, added bool) { - if added { - announces.Add(1) - } else { - announces.Add(-1) - } - } - // Create a valid chain and an infinite junk chain - targetBlocks := hashLimit + 2*maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - validHeaderFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - validBodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - attack, _ := makeChain(targetBlocks, 0, unknownBlock) - attackerHeaderFetcher := tester.makeHeaderFetcher("attacker", nil, -gatherSlack) - attackerBodyFetcher := tester.makeBodyFetcher("attacker", nil, 0) - - // Feed the tester a huge hashset from the attacker, and a limited from the valid peer - for i := 0; i < len(attack); i++ { - if i < maxQueueDist { - tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), validHeaderFetcher, validBodyFetcher) - } - tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), attackerHeaderFetcher, attackerBodyFetcher) - } - if count := announces.Load(); count != hashLimit+maxQueueDist { - t.Fatalf("queued announce count mismatch: have %d, want %d", count, hashLimit+maxQueueDist) - } - // Wait for fetches to complete - verifyImportCount(t, imported, maxQueueDist) - - // Feed the remaining valid hashes to ensure DOS protection state remains clean - for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), validHeaderFetcher, validBodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} - -// Tests that blocks sent to the fetcher (either through propagation or via hash -// announces and retrievals) don't pile up indefinitely, exhausting available -// system memory. -func TestBlockMemoryExhaustionAttack(t *testing.T) { - // Create a tester with instrumented import hooks - tester := newTester(false) - - imported, enqueued := make(chan interface{}), atomic.Int32{} - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - tester.fetcher.queueChangeHook = func(hash common.Hash, added bool) { - if added { - enqueued.Add(1) - } else { - enqueued.Add(-1) - } - } - // Create a valid chain and a batch of dangling (but in range) blocks - targetBlocks := hashLimit + 2*maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - attack := make(map[common.Hash]*types.Block) - for i := byte(0); len(attack) < blockLimit+2*maxQueueDist; i++ { - hashes, blocks := makeChain(maxQueueDist-1, i, unknownBlock) - for _, hash := range hashes[:maxQueueDist-2] { - attack[hash] = blocks[hash] - } - } - // Try to feed all the attacker blocks make sure only a limited batch is accepted - for _, block := range attack { - tester.fetcher.Enqueue("attacker", block) - } - time.Sleep(200 * time.Millisecond) - if queued := enqueued.Load(); queued != blockLimit { - t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit) - } - // Queue up a batch of valid blocks, and check that a new peer is allowed to do so - for i := 0; i < maxQueueDist-1; i++ { - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-3-i]]) - } - time.Sleep(100 * time.Millisecond) - if queued := enqueued.Load(); queued != blockLimit+maxQueueDist-1 { - t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit+maxQueueDist-1) - } - // Insert the missing piece (and sanity check the import) - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2]]) - verifyImportCount(t, imported, maxQueueDist) - - // Insert the remaining blocks in chunks to ensure clean DOS protection - for i := maxQueueDist; i < len(hashes)-1; i++ { - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2-i]]) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index ea7892d8d84e..18c5ff007a9f 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -107,6 +107,8 @@ var ( txFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/fetching/hashes", nil) ) +var errTerminated = errors.New("terminated") + // txAnnounce is the notification of the availability of a batch // of new transactions in the network. type txAnnounce struct { @@ -783,7 +785,7 @@ func (f *TxFetcher) loop() { // rescheduleWait iterates over all the transactions currently in the waitlist // and schedules the movement into the fetcher for the earliest. // -// The method has a granularity of 'gatherSlack', since there's not much point in +// The method has a granularity of 'txGatherSlack', since there's not much point in // spinning over all the transactions just to maybe find one that should trigger // a few ms earlier. func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { @@ -796,7 +798,7 @@ func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { for _, instance := range f.waittime { if earliest > instance { earliest = instance - if txArriveTimeout-time.Duration(now-earliest) < gatherSlack { + if txArriveTimeout-time.Duration(now-earliest) < txGatherSlack { break } } @@ -809,7 +811,7 @@ func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { // rescheduleTimeout iterates over all the transactions currently in flight and // schedules a cleanup run when the first would trigger. // -// The method has a granularity of 'gatherSlack', since there's not much point in +// The method has a granularity of 'txGatherSlack', since there's not much point in // spinning over all the transactions just to maybe find one that should trigger // a few ms earlier. // @@ -834,7 +836,7 @@ func (f *TxFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{} } if earliest > req.time { earliest = req.time - if txFetchTimeout-time.Duration(now-earliest) < gatherSlack { + if txFetchTimeout-time.Duration(now-earliest) < txGatherSlack { break } } diff --git a/eth/handler.go b/eth/handler.go index bc27eb4b88c5..a32a04e00b72 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -25,8 +25,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/rawdb" @@ -91,7 +89,6 @@ type handlerConfig struct { Database ethdb.Database // Database for direct sync insertions Chain *core.BlockChain // Blockchain to serve data from TxPool txPool // Transaction pool to propagate from - Merger *consensus.Merger // The manager for eth1/2 transition Network uint64 // Network identifier to advertise Sync downloader.SyncMode // Whether to snap or full sync BloomCache uint64 // Megabytes to alloc for snap sync bloom @@ -112,24 +109,20 @@ type handler struct { chain *core.BlockChain maxPeers int - downloader *downloader.Downloader - blockFetcher *fetcher.BlockFetcher - txFetcher *fetcher.TxFetcher - peers *peerSet - merger *consensus.Merger + downloader *downloader.Downloader + txFetcher *fetcher.TxFetcher + peers *peerSet - eventMux *event.TypeMux - txsCh chan core.NewTxsEvent - txsSub event.Subscription - minedBlockSub *event.TypeMuxSubscription + eventMux *event.TypeMux + txsCh chan core.NewTxsEvent + txsSub event.Subscription requiredBlocks map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop quitSync chan struct{} - chainSync *chainSyncer - wg sync.WaitGroup + wg sync.WaitGroup handlerStartCh chan struct{} handlerDoneCh chan struct{} @@ -150,7 +143,6 @@ func newHandler(config *handlerConfig) (*handler, error) { txpool: config.TxPool, chain: config.Chain, peers: newPeerSet(), - merger: config.Merger, requiredBlocks: config.RequiredBlocks, quitSync: make(chan struct{}), handlerDoneCh: make(chan struct{}), @@ -190,92 +182,6 @@ func newHandler(config *handlerConfig) (*handler, error) { } // Construct the downloader (long sync) h.downloader = downloader.New(config.Database, h.eventMux, h.chain, nil, h.removePeer, h.enableSyncedFeatures) - if ttd := h.chain.Config().TerminalTotalDifficulty; ttd != nil { - if h.chain.Config().TerminalTotalDifficultyPassed { - log.Info("Chain post-merge, sync via beacon client") - } else { - head := h.chain.CurrentBlock() - if td := h.chain.GetTd(head.Hash(), head.Number.Uint64()); td.Cmp(ttd) >= 0 { - log.Info("Chain post-TTD, sync via beacon client") - } else { - log.Warn("Chain pre-merge, sync via PoW (ensure beacon client is ready)") - } - } - } else if h.chain.Config().TerminalTotalDifficultyPassed { - log.Error("Chain configured post-merge, but without TTD. Are you debugging sync?") - } - // Construct the fetcher (short sync) - validator := func(header *types.Header) error { - // All the block fetcher activities should be disabled - // after the transition. Print the warning log. - if h.merger.PoSFinalized() { - log.Warn("Unexpected validation activity", "hash", header.Hash(), "number", header.Number) - return errors.New("unexpected behavior after transition") - } - // Reject all the PoS style headers in the first place. No matter - // the chain has finished the transition or not, the PoS headers - // should only come from the trusted consensus layer instead of - // p2p network. - if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { - if beacon.IsPoSHeader(header) { - return errors.New("unexpected post-merge header") - } - } - return h.chain.Engine().VerifyHeader(h.chain, header) - } - heighter := func() uint64 { - return h.chain.CurrentBlock().Number.Uint64() - } - inserter := func(blocks types.Blocks) (int, error) { - // All the block fetcher activities should be disabled - // after the transition. Print the warning log. - if h.merger.PoSFinalized() { - var ctx []interface{} - ctx = append(ctx, "blocks", len(blocks)) - if len(blocks) > 0 { - ctx = append(ctx, "firsthash", blocks[0].Hash()) - ctx = append(ctx, "firstnumber", blocks[0].Number()) - ctx = append(ctx, "lasthash", blocks[len(blocks)-1].Hash()) - ctx = append(ctx, "lastnumber", blocks[len(blocks)-1].Number()) - } - log.Warn("Unexpected insertion activity", ctx...) - return 0, errors.New("unexpected behavior after transition") - } - // If snap sync is running, deny importing weird blocks. This is a problematic - // clause when starting up a new network, because snap-syncing miners might not - // accept each others' blocks until a restart. Unfortunately we haven't figured - // out a way yet where nodes can decide unilaterally whether the network is new - // or not. This should be fixed if we figure out a solution. - if !h.synced.Load() { - log.Warn("Syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) - return 0, nil - } - if h.merger.TDDReached() { - // The blocks from the p2p network is regarded as untrusted - // after the transition. In theory block gossip should be disabled - // entirely whenever the transition is started. But in order to - // handle the transition boundary reorg in the consensus-layer, - // the legacy blocks are still accepted, but only for the terminal - // pow blocks. Spec: https://github.com/ethereum/EIPs/blob/master/EIPS/eip-3675.md#halt-the-importing-of-pow-blocks - for i, block := range blocks { - ptd := h.chain.GetTd(block.ParentHash(), block.NumberU64()-1) - if ptd == nil { - return 0, nil - } - td := new(big.Int).Add(ptd, block.Difficulty()) - if !h.chain.Config().IsTerminalPoWBlock(ptd, td) { - log.Info("Filtered out non-terminal pow block", "number", block.NumberU64(), "hash", block.Hash()) - return 0, nil - } - if err := h.chain.InsertBlockWithoutSetHead(block); err != nil { - return i, err - } - } - return 0, nil - } - return h.chain.InsertChain(blocks) - } - h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock, heighter, nil, inserter, h.removePeer) fetchTx := func(peer string, hashes []common.Hash) error { p := h.peers.peer(peer) @@ -288,7 +194,6 @@ func newHandler(config *handlerConfig) (*handler, error) { return h.txpool.Add(txs, false, false) } h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, addTxs, fetchTx, h.removePeer) - h.chainSync = newChainSyncer(h) return h, nil } @@ -398,8 +303,6 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { return err } } - h.chainSync.handlePeerEvent() - // Propagate existing transactions. new transactions appearing // after this will be sent via broadcasts. h.syncTransactions(peer) @@ -526,14 +429,8 @@ func (h *handler) Start(maxPeers int) { h.txsSub = h.txpool.SubscribeTransactions(h.txsCh, false) go h.txBroadcastLoop() - // broadcast mined blocks - h.wg.Add(1) - h.minedBlockSub = h.eventMux.Subscribe(core.NewMinedBlockEvent{}) - go h.minedBroadcastLoop() - // start sync handlers - h.wg.Add(1) - go h.chainSync.loop() + h.txFetcher.Start() // start peer handler tracker h.wg.Add(1) @@ -541,8 +438,9 @@ func (h *handler) Start(maxPeers int) { } func (h *handler) Stop() { - h.txsSub.Unsubscribe() // quits txBroadcastLoop - h.minedBlockSub.Unsubscribe() // quits blockBroadcastLoop + h.txsSub.Unsubscribe() // quits txBroadcastLoop + h.txFetcher.Stop() + h.downloader.Terminate() // Quit chainSync and txsync64. // After this is done, no new peers will be accepted. @@ -558,50 +456,6 @@ func (h *handler) Stop() { log.Info("Ethereum protocol stopped") } -// BroadcastBlock will either propagate a block to a subset of its peers, or -// will only announce its availability (depending what's requested). -func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { - // Disable the block propagation if the chain has already entered the PoS - // stage. The block propagation is delegated to the consensus layer. - if h.merger.PoSFinalized() { - return - } - // Disable the block propagation if it's the post-merge block. - if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { - if beacon.IsPoSHeader(block.Header()) { - return - } - } - hash := block.Hash() - peers := h.peers.peersWithoutBlock(hash) - - // If propagation is requested, send to a subset of the peer - if propagate { - // Calculate the TD of the block (it's not imported yet, so block.Td is not valid) - var td *big.Int - if parent := h.chain.GetBlock(block.ParentHash(), block.NumberU64()-1); parent != nil { - td = new(big.Int).Add(block.Difficulty(), h.chain.GetTd(block.ParentHash(), block.NumberU64()-1)) - } else { - log.Error("Propagating dangling block", "number", block.Number(), "hash", hash) - return - } - // Send the block to a subset of our peers - transfer := peers[:int(math.Sqrt(float64(len(peers))))] - for _, peer := range transfer { - peer.AsyncSendNewBlock(block, td) - } - log.Trace("Propagated block", "hash", hash, "recipients", len(transfer), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) - return - } - // Otherwise if the block is indeed in out own chain, announce it - if h.chain.HasBlock(hash, block.NumberU64()) { - for _, peer := range peers { - peer.AsyncSendNewBlockHash(block) - } - log.Trace("Announced block", "hash", hash, "recipients", len(peers), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) - } -} - // BroadcastTransactions will propagate a batch of transactions // - To a square root of all peers for non-blob transactions // - And, separately, as announcements to all peers which are not known to @@ -684,18 +538,6 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { "bcastpeers", directPeers, "bcastcount", directCount, "annpeers", annPeers, "anncount", annCount) } -// minedBroadcastLoop sends mined blocks to connected peers. -func (h *handler) minedBroadcastLoop() { - defer h.wg.Done() - - for obj := range h.minedBlockSub.Chan() { - if ev, ok := obj.Data.(core.NewMinedBlockEvent); ok { - h.BroadcastBlock(ev.Block, true) // First propagate block to peers - h.BroadcastBlock(ev.Block, false) // Only then announce to the rest - } - } -} - // txBroadcastLoop announces new transactions to connected peers. func (h *handler) txBroadcastLoop() { defer h.wg.Done() diff --git a/eth/handler_eth.go b/eth/handler_eth.go index f1284c10e637..b2cd52a22105 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -19,10 +19,7 @@ package eth import ( "errors" "fmt" - "math/big" - "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -60,13 +57,6 @@ func (h *ethHandler) AcceptTxs() bool { func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Consume any broadcasts and announces, forwarding the rest to the downloader switch packet := packet.(type) { - case *eth.NewBlockHashesPacket: - hashes, numbers := packet.Unpack() - return h.handleBlockAnnounces(peer, hashes, numbers) - - case *eth.NewBlockPacket: - return h.handleBlockBroadcast(peer, packet.Block, packet.TD) - case *eth.NewPooledTransactionHashesPacket: return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes) @@ -85,55 +75,3 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { return fmt.Errorf("unexpected eth packet type: %T", packet) } } - -// handleBlockAnnounces is invoked from a peer's message handler when it transmits a -// batch of block announcements for the local node to process. -func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, numbers []uint64) error { - // Drop all incoming block announces from the p2p network if - // the chain already entered the pos stage and disconnect the - // remote peer. - if h.merger.PoSFinalized() { - return errors.New("disallowed block announcement") - } - // Schedule all the unknown hashes for retrieval - var ( - unknownHashes = make([]common.Hash, 0, len(hashes)) - unknownNumbers = make([]uint64, 0, len(numbers)) - ) - for i := 0; i < len(hashes); i++ { - if !h.chain.HasBlock(hashes[i], numbers[i]) { - unknownHashes = append(unknownHashes, hashes[i]) - unknownNumbers = append(unknownNumbers, numbers[i]) - } - } - for i := 0; i < len(unknownHashes); i++ { - h.blockFetcher.Notify(peer.ID(), unknownHashes[i], unknownNumbers[i], time.Now(), peer.RequestOneHeader, peer.RequestBodies) - } - return nil -} - -// handleBlockBroadcast is invoked from a peer's message handler when it transmits a -// block broadcast for the local node to process. -func (h *ethHandler) handleBlockBroadcast(peer *eth.Peer, block *types.Block, td *big.Int) error { - // Drop all incoming block announces from the p2p network if - // the chain already entered the pos stage and disconnect the - // remote peer. - if h.merger.PoSFinalized() { - return errors.New("disallowed block broadcast") - } - // Schedule the block for import - h.blockFetcher.Enqueue(peer.ID(), block) - - // Assuming the block is importable by the peer, but possibly not yet done so, - // calculate the head hash and TD that the peer truly must have. - var ( - trueHead = block.ParentHash() - trueTD = new(big.Int).Sub(td, block.Difficulty()) - ) - // Update the peer's total difficulty if better than the previous - if _, td := peer.Head(); trueTD.Cmp(td) > 0 { - peer.SetHead(trueHead, trueTD) - h.chainSync.handlePeerEvent() - } - return nil -} diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 579ca3c09735..297a6bf154ed 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" @@ -109,7 +108,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { Database: dbNoFork, Chain: chainNoFork, TxPool: newTestTxPool(), - Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FullSync, BloomCache: 1, @@ -118,7 +116,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { Database: dbProFork, Chain: chainProFork, TxPool: newTestTxPool(), - Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FullSync, BloomCache: 1, @@ -441,159 +438,3 @@ func testTransactionPropagation(t *testing.T, protocol uint) { } } } - -// Tests that blocks are broadcast to a sqrt number of peers only. -func TestBroadcastBlock1Peer(t *testing.T) { testBroadcastBlock(t, 1, 1) } -func TestBroadcastBlock2Peers(t *testing.T) { testBroadcastBlock(t, 2, 1) } -func TestBroadcastBlock3Peers(t *testing.T) { testBroadcastBlock(t, 3, 1) } -func TestBroadcastBlock4Peers(t *testing.T) { testBroadcastBlock(t, 4, 2) } -func TestBroadcastBlock5Peers(t *testing.T) { testBroadcastBlock(t, 5, 2) } -func TestBroadcastBlock8Peers(t *testing.T) { testBroadcastBlock(t, 9, 3) } -func TestBroadcastBlock12Peers(t *testing.T) { testBroadcastBlock(t, 12, 3) } -func TestBroadcastBlock16Peers(t *testing.T) { testBroadcastBlock(t, 16, 4) } -func TestBroadcastBloc26Peers(t *testing.T) { testBroadcastBlock(t, 26, 5) } -func TestBroadcastBlock100Peers(t *testing.T) { testBroadcastBlock(t, 100, 10) } - -func testBroadcastBlock(t *testing.T, peers, bcasts int) { - t.Parallel() - - // Create a source handler to broadcast blocks from and a number of sinks - // to receive them. - source := newTestHandlerWithBlocks(1) - defer source.close() - - sinks := make([]*testEthHandler, peers) - for i := 0; i < len(sinks); i++ { - sinks[i] = new(testEthHandler) - } - // Interconnect all the sink handlers with the source handler - var ( - genesis = source.chain.Genesis() - td = source.chain.GetTd(genesis.Hash(), genesis.NumberU64()) - ) - for i, sink := range sinks { - sink := sink // Closure for gorotuine below - - sourcePipe, sinkPipe := p2p.MsgPipe() - defer sourcePipe.Close() - defer sinkPipe.Close() - - sourcePeer := eth.NewPeer(eth.ETH68, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH68, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) - defer sourcePeer.Close() - defer sinkPeer.Close() - - go source.handler.runEthPeer(sourcePeer, func(peer *eth.Peer) error { - return eth.Handle((*ethHandler)(source.handler), peer) - }) - if err := sinkPeer.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain)); err != nil { - t.Fatalf("failed to run protocol handshake") - } - go eth.Handle(sink, sinkPeer) - } - // Subscribe to all the transaction pools - blockChs := make([]chan *types.Block, len(sinks)) - for i := 0; i < len(sinks); i++ { - blockChs[i] = make(chan *types.Block, 1) - defer close(blockChs[i]) - - sub := sinks[i].blockBroadcasts.Subscribe(blockChs[i]) - defer sub.Unsubscribe() - } - // Initiate a block propagation across the peers - time.Sleep(100 * time.Millisecond) - header := source.chain.CurrentBlock() - source.handler.BroadcastBlock(source.chain.GetBlock(header.Hash(), header.Number.Uint64()), true) - - // Iterate through all the sinks and ensure the correct number got the block - done := make(chan struct{}, peers) - for _, ch := range blockChs { - ch := ch - go func() { - <-ch - done <- struct{}{} - }() - } - var received int - for { - select { - case <-done: - received++ - - case <-time.After(100 * time.Millisecond): - if received != bcasts { - t.Errorf("broadcast count mismatch: have %d, want %d", received, bcasts) - } - return - } - } -} - -// Tests that a propagated malformed block (uncles or transactions don't match -// with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock68(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH68) } - -func testBroadcastMalformedBlock(t *testing.T, protocol uint) { - t.Parallel() - - // Create a source handler to broadcast blocks from and a number of sinks - // to receive them. - source := newTestHandlerWithBlocks(1) - defer source.close() - - // Create a source handler to send messages through and a sink peer to receive them - p2pSrc, p2pSink := p2p.MsgPipe() - defer p2pSrc.Close() - defer p2pSink.Close() - - src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, source.txpool) - sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, source.txpool) - defer src.Close() - defer sink.Close() - - go source.handler.runEthPeer(src, func(peer *eth.Peer) error { - return eth.Handle((*ethHandler)(source.handler), peer) - }) - // Run the handshake locally to avoid spinning up a sink handler - var ( - genesis = source.chain.Genesis() - td = source.chain.GetTd(genesis.Hash(), genesis.NumberU64()) - ) - if err := sink.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain)); err != nil { - t.Fatalf("failed to run protocol handshake") - } - // After the handshake completes, the source handler should stream the sink - // the blocks, subscribe to inbound network events - backend := new(testEthHandler) - - blocks := make(chan *types.Block, 1) - sub := backend.blockBroadcasts.Subscribe(blocks) - defer sub.Unsubscribe() - - go eth.Handle(backend, sink) - - // Create various combinations of malformed blocks - head := source.chain.CurrentBlock() - block := source.chain.GetBlock(head.Hash(), head.Number.Uint64()) - - malformedUncles := head - malformedUncles.UncleHash[0]++ - malformedTransactions := head - malformedTransactions.TxHash[0]++ - malformedEverything := head - malformedEverything.UncleHash[0]++ - malformedEverything.TxHash[0]++ - - // Try to broadcast all malformations and ensure they all get discarded - for _, header := range []*types.Header{malformedUncles, malformedTransactions, malformedEverything} { - block := types.NewBlockWithHeader(header).WithBody(block.Transactions(), block.Uncles()) - if err := src.SendNewBlock(block, big.NewInt(131136)); err != nil { - t.Fatalf("failed to broadcast block: %v", err) - } - select { - case <-blocks: - t.Fatalf("malformed block forwarded") - case <-time.After(100 * time.Millisecond): - } - } -} diff --git a/eth/handler_test.go b/eth/handler_test.go index 58353f6b6452..bcc8ea30e415 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -22,7 +22,6 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -164,7 +163,6 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Database: db, Chain: chain, TxPool: txpool, - Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.SnapSync, BloomCache: 1, diff --git a/eth/peerset.go b/eth/peerset.go index c56a7223e964..6b0aff226c1c 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -19,7 +19,6 @@ package eth import ( "errors" "fmt" - "math/big" "sync" "github.com/ethereum/go-ethereum/common" @@ -192,21 +191,6 @@ func (ps *peerSet) peer(id string) *ethPeer { return ps.peers[id] } -// peersWithoutBlock retrieves a list of peers that do not have a given block in -// their set of known hashes so it might be propagated to them. -func (ps *peerSet) peersWithoutBlock(hash common.Hash) []*ethPeer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - list := make([]*ethPeer, 0, len(ps.peers)) - for _, p := range ps.peers { - if !p.KnownBlock(hash) { - list = append(list, p) - } - } - return list -} - // peersWithoutTransaction retrieves a list of peers that do not have a given // transaction in their set of known hashes. func (ps *peerSet) peersWithoutTransaction(hash common.Hash) []*ethPeer { @@ -240,24 +224,6 @@ func (ps *peerSet) snapLen() int { return ps.snapPeers } -// peerWithHighestTD retrieves the known peer with the currently highest total -// difficulty, but below the given PoS switchover threshold. -func (ps *peerSet) peerWithHighestTD() *eth.Peer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - var ( - bestPeer *eth.Peer - bestTd *big.Int - ) - for _, p := range ps.peers { - if _, td := p.Head(); bestPeer == nil || td.Cmp(bestTd) > 0 { - bestPeer, bestTd = p.Peer, td - } - } - return bestPeer -} - // close disconnects all peers. func (ps *peerSet) close() { ps.lock.Lock() diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index ad5395cb8dd9..f0ed1d6bc9a0 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -17,8 +17,6 @@ package eth import ( - "math/big" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" ) @@ -29,37 +27,6 @@ const ( maxTxPacketSize = 100 * 1024 ) -// blockPropagation is a block propagation event, waiting for its turn in the -// broadcast queue. -type blockPropagation struct { - block *types.Block - td *big.Int -} - -// broadcastBlocks is a write loop that multiplexes blocks and block announcements -// to the remote peer. The goal is to have an async writer that does not lock up -// node internals and at the same time rate limits queued data. -func (p *Peer) broadcastBlocks() { - for { - select { - case prop := <-p.queuedBlocks: - if err := p.SendNewBlock(prop.block, prop.td); err != nil { - return - } - p.Log().Trace("Propagated block", "number", prop.block.Number(), "hash", prop.block.Hash(), "td", prop.td) - - case block := <-p.queuedBlockAnns: - if err := p.SendNewBlockHashes([]common.Hash{block.Hash()}, []uint64{block.NumberU64()}); err != nil { - return - } - p.Log().Trace("Announced block", "number", block.Number(), "hash", block.Hash()) - - case <-p.term: - return - } - } -} - // broadcastTransactions is a write loop that schedules transaction broadcasts // to the remote peer. The goal is to have an async writer that does not lock up // node internals and at the same time rate limits queued data. diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 0275708a6cd5..96656afb1b20 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -18,6 +18,7 @@ package eth import ( "encoding/json" + "errors" "fmt" "github.com/ethereum/go-ethereum/common" @@ -274,43 +275,11 @@ func ServiceGetReceiptsQuery(chain *core.BlockChain, query GetReceiptsRequest) [ } func handleNewBlockhashes(backend Backend, msg Decoder, peer *Peer) error { - // A batch of new block announcements just arrived - ann := new(NewBlockHashesPacket) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Mark the hashes as present at the remote node - for _, block := range *ann { - peer.markBlock(block.Hash) - } - // Deliver them all to the backend for queuing - return backend.Handle(peer, ann) + return errors.New("block announcements disallowed") // We dropped support for non-merge networks } func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { - // Retrieve and decode the propagated block - ann := new(NewBlockPacket) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - if err := ann.sanityCheck(); err != nil { - return err - } - if hash := types.CalcUncleHash(ann.Block.Uncles()); hash != ann.Block.UncleHash() { - log.Warn("Propagated block has invalid uncles", "have", hash, "exp", ann.Block.UncleHash()) - return nil // TODO(karalabe): return error eventually, but wait a few releases - } - if hash := types.DeriveSha(ann.Block.Transactions(), trie.NewStackTrie(nil)); hash != ann.Block.TxHash() { - log.Warn("Propagated block has invalid body", "have", hash, "exp", ann.Block.TxHash()) - return nil // TODO(karalabe): return error eventually, but wait a few releases - } - ann.Block.ReceivedAt = msg.Time() - ann.Block.ReceivedFrom = peer - - // Mark the peer as owning the block - peer.markBlock(ann.Block.Hash()) - - return backend.Handle(peer, ann) + return errors.New("block broadcasts disallowed") // We dropped support for non-merge networks } func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index ffd78b05946a..94f28f240f86 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -33,10 +33,6 @@ const ( // before starting to randomly evict them. maxKnownTxs = 32768 - // maxKnownBlocks is the maximum block hashes to keep in the known list - // before starting to randomly evict them. - maxKnownBlocks = 1024 - // maxQueuedTxs is the maximum number of transactions to queue up before dropping // older broadcasts. maxQueuedTxs = 4096 @@ -44,16 +40,6 @@ const ( // maxQueuedTxAnns is the maximum number of transaction announcements to queue up // before dropping older announcements. maxQueuedTxAnns = 4096 - - // maxQueuedBlocks is the maximum number of block propagations to queue up before - // dropping broadcasts. There's not much point in queueing stale blocks, so a few - // that might cover uncles should be enough. - maxQueuedBlocks = 4 - - // maxQueuedBlockAnns is the maximum number of block announcements to queue up before - // dropping broadcasts. Similarly to block propagations, there's no point to queue - // above some healthy uncle limit, so use that. - maxQueuedBlockAnns = 4 ) // max is a helper function which returns the larger of the two given integers. @@ -75,10 +61,6 @@ type Peer struct { head common.Hash // Latest advertised head block hash td *big.Int // Latest advertised head block total difficulty - knownBlocks *knownCache // Set of block hashes known to be known by this peer - queuedBlocks chan *blockPropagation // Queue of blocks to broadcast to the peer - queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer - txpool TxPool // Transaction pool used by the broadcasters for liveness checks knownTxs *knownCache // Set of transaction hashes known to be known by this peer txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests @@ -96,24 +78,20 @@ type Peer struct { // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Peer { peer := &Peer{ - id: p.ID().String(), - Peer: p, - rw: rw, - version: version, - knownTxs: newKnownCache(maxKnownTxs), - knownBlocks: newKnownCache(maxKnownBlocks), - queuedBlocks: make(chan *blockPropagation, maxQueuedBlocks), - queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns), - txBroadcast: make(chan []common.Hash), - txAnnounce: make(chan []common.Hash), - reqDispatch: make(chan *request), - reqCancel: make(chan *cancel), - resDispatch: make(chan *response), - txpool: txpool, - term: make(chan struct{}), + id: p.ID().String(), + Peer: p, + rw: rw, + version: version, + knownTxs: newKnownCache(maxKnownTxs), + txBroadcast: make(chan []common.Hash), + txAnnounce: make(chan []common.Hash), + reqDispatch: make(chan *request), + reqCancel: make(chan *cancel), + resDispatch: make(chan *response), + txpool: txpool, + term: make(chan struct{}), } // Start up all the broadcasters - go peer.broadcastBlocks() go peer.broadcastTransactions() go peer.announceTransactions() go peer.dispatcher() @@ -156,23 +134,11 @@ func (p *Peer) SetHead(hash common.Hash, td *big.Int) { p.td.Set(td) } -// KnownBlock returns whether peer is known to already have a block. -func (p *Peer) KnownBlock(hash common.Hash) bool { - return p.knownBlocks.Contains(hash) -} - // KnownTransaction returns whether peer is known to already have a transaction. func (p *Peer) KnownTransaction(hash common.Hash) bool { return p.knownTxs.Contains(hash) } -// markBlock marks a block as known for the peer, ensuring that the block will -// never be propagated to this particular peer. -func (p *Peer) markBlock(hash common.Hash) { - // If we reached the memory allowance, drop a previously known block hash - p.knownBlocks.Add(hash) -} - // markTransaction marks a transaction as known for the peer, ensuring that it // will never be propagated to this particular peer. func (p *Peer) markTransaction(hash common.Hash) { @@ -248,55 +214,6 @@ func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs [ }) } -// SendNewBlockHashes announces the availability of a number of blocks through -// a hash notification. -func (p *Peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { - // Mark all the block hashes as known, but ensure we don't overflow our limits - p.knownBlocks.Add(hashes...) - - request := make(NewBlockHashesPacket, len(hashes)) - for i := 0; i < len(hashes); i++ { - request[i].Hash = hashes[i] - request[i].Number = numbers[i] - } - return p2p.Send(p.rw, NewBlockHashesMsg, request) -} - -// AsyncSendNewBlockHash queues the availability of a block for propagation to a -// remote peer. If the peer's broadcast queue is full, the event is silently -// dropped. -func (p *Peer) AsyncSendNewBlockHash(block *types.Block) { - select { - case p.queuedBlockAnns <- block: - // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) - default: - p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash()) - } -} - -// SendNewBlock propagates an entire block to a remote peer. -func (p *Peer) SendNewBlock(block *types.Block, td *big.Int) error { - // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) - return p2p.Send(p.rw, NewBlockMsg, &NewBlockPacket{ - Block: block, - TD: td, - }) -} - -// AsyncSendNewBlock queues an entire block for propagation to a remote peer. If -// the peer's broadcast queue is full, the event is silently dropped. -func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { - select { - case p.queuedBlocks <- &blockPropagation{block: block, td: td}: - // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) - default: - p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash()) - } -} - // ReplyBlockHeadersRLP is the response to GetBlockHeaders. func (p *Peer) ReplyBlockHeadersRLP(id uint64, headers []rlp.RawValue) error { return p2p.Send(p.rw, BlockHeadersMsg, &BlockHeadersRLPPacket{ diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 47e8d97244cb..c5cb2dd1dca4 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -189,19 +189,6 @@ type NewBlockPacket struct { TD *big.Int } -// sanityCheck verifies that the values are reasonable, as a DoS protection -func (request *NewBlockPacket) sanityCheck() error { - if err := request.Block.SanityCheck(); err != nil { - return err - } - //TD at mainnet block #7753254 is 76 bits. If it becomes 100 million times - // larger, it will still fit within 100 bits - if tdlen := request.TD.BitLen(); tdlen > 100 { - return fmt.Errorf("too large block TD: bitlen %d", tdlen) - } - return nil -} - // GetBlockBodiesRequest represents a block body query. type GetBlockBodiesRequest []common.Hash diff --git a/eth/sync.go b/eth/sync.go index cdcfbdb3db49..61f2b2b376cd 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -17,21 +17,9 @@ package eth import ( - "errors" - "math/big" - "time" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/log" -) - -const ( - forceSyncCycle = 10 * time.Second // Time interval to force syncs, even if few peers are available - defaultMinSyncPeers = 5 // Amount of peers desired to start syncing ) // syncTransactions starts sending all currently pending transactions to the given peer. @@ -47,206 +35,3 @@ func (h *handler) syncTransactions(p *eth.Peer) { } p.AsyncSendPooledTransactionHashes(hashes) } - -// chainSyncer coordinates blockchain sync components. -type chainSyncer struct { - handler *handler - force *time.Timer - forced bool // true when force timer fired - warned time.Time - peerEventCh chan struct{} - doneCh chan error // non-nil when sync is running -} - -// chainSyncOp is a scheduled sync operation. -type chainSyncOp struct { - mode downloader.SyncMode - peer *eth.Peer - td *big.Int - head common.Hash -} - -// newChainSyncer creates a chainSyncer. -func newChainSyncer(handler *handler) *chainSyncer { - return &chainSyncer{ - handler: handler, - peerEventCh: make(chan struct{}), - } -} - -// handlePeerEvent notifies the syncer about a change in the peer set. -// This is called for new peers and every time a peer announces a new -// chain head. -func (cs *chainSyncer) handlePeerEvent() bool { - select { - case cs.peerEventCh <- struct{}{}: - return true - case <-cs.handler.quitSync: - return false - } -} - -// loop runs in its own goroutine and launches the sync when necessary. -func (cs *chainSyncer) loop() { - defer cs.handler.wg.Done() - - cs.handler.blockFetcher.Start() - cs.handler.txFetcher.Start() - defer cs.handler.blockFetcher.Stop() - defer cs.handler.txFetcher.Stop() - defer cs.handler.downloader.Terminate() - - // The force timer lowers the peer count threshold down to one when it fires. - // This ensures we'll always start sync even if there aren't enough peers. - cs.force = time.NewTimer(forceSyncCycle) - defer cs.force.Stop() - - for { - if op := cs.nextSyncOp(); op != nil { - cs.startSync(op) - } - select { - case <-cs.peerEventCh: - // Peer information changed, recheck. - case err := <-cs.doneCh: - cs.doneCh = nil - cs.force.Reset(forceSyncCycle) - cs.forced = false - - // If we've reached the merge transition but no beacon client is available, or - // it has not yet switched us over, keep warning the user that their infra is - // potentially flaky. - if errors.Is(err, downloader.ErrMergeTransition) && time.Since(cs.warned) > 10*time.Second { - log.Warn("Local chain is post-merge, waiting for beacon client sync switch-over...") - cs.warned = time.Now() - } - case <-cs.force.C: - cs.forced = true - - case <-cs.handler.quitSync: - // Disable all insertion on the blockchain. This needs to happen before - // terminating the downloader because the downloader waits for blockchain - // inserts, and these can take a long time to finish. - cs.handler.chain.StopInsert() - cs.handler.downloader.Terminate() - if cs.doneCh != nil { - <-cs.doneCh - } - return - } - } -} - -// nextSyncOp determines whether sync is required at this time. -func (cs *chainSyncer) nextSyncOp() *chainSyncOp { - if cs.doneCh != nil { - return nil // Sync already running - } - // If a beacon client once took over control, disable the entire legacy sync - // path from here on end. Note, there is a slight "race" between reaching TTD - // and the beacon client taking over. The downloader will enforce that nothing - // above the first TTD will be delivered to the chain for import. - // - // An alternative would be to check the local chain for exceeding the TTD and - // avoid triggering a sync in that case, but that could also miss sibling or - // other family TTD block being accepted. - if cs.handler.chain.Config().TerminalTotalDifficultyPassed || cs.handler.merger.TDDReached() { - return nil - } - // Ensure we're at minimum peer count. - minPeers := defaultMinSyncPeers - if cs.forced { - minPeers = 1 - } else if minPeers > cs.handler.maxPeers { - minPeers = cs.handler.maxPeers - } - if cs.handler.peers.len() < minPeers { - return nil - } - // We have enough peers, pick the one with the highest TD, but avoid going - // over the terminal total difficulty. Above that we expect the consensus - // clients to direct the chain head to sync to. - peer := cs.handler.peers.peerWithHighestTD() - if peer == nil { - return nil - } - mode, ourTD := cs.modeAndLocalHead() - op := peerToSyncOp(mode, peer) - if op.td.Cmp(ourTD) <= 0 { - // We seem to be in sync according to the legacy rules. In the merge - // world, it can also mean we're stuck on the merge block, waiting for - // a beacon client. In the latter case, notify the user. - if ttd := cs.handler.chain.Config().TerminalTotalDifficulty; ttd != nil && ourTD.Cmp(ttd) >= 0 && time.Since(cs.warned) > 10*time.Second { - log.Warn("Local chain is post-merge, waiting for beacon client sync switch-over...") - cs.warned = time.Now() - } - return nil // We're in sync - } - return op -} - -func peerToSyncOp(mode downloader.SyncMode, p *eth.Peer) *chainSyncOp { - peerHead, peerTD := p.Head() - return &chainSyncOp{mode: mode, peer: p, td: peerTD, head: peerHead} -} - -func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) { - // If we're in snap sync mode, return that directly - if cs.handler.snapSync.Load() { - block := cs.handler.chain.CurrentSnapBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) - return downloader.SnapSync, td - } - // We are probably in full sync, but we might have rewound to before the - // snap sync pivot, check if we should re-enable snap sync. - head := cs.handler.chain.CurrentBlock() - if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil { - if head.Number.Uint64() < *pivot { - block := cs.handler.chain.CurrentSnapBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) - return downloader.SnapSync, td - } - } - // We are in a full sync, but the associated head state is missing. To complete - // the head state, forcefully rerun the snap sync. Note it doesn't mean the - // persistent state is corrupted, just mismatch with the head block. - if !cs.handler.chain.HasState(head.Root) { - block := cs.handler.chain.CurrentSnapBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) - log.Info("Reenabled snap sync as chain is stateless") - return downloader.SnapSync, td - } - // Nope, we're really full syncing - td := cs.handler.chain.GetTd(head.Hash(), head.Number.Uint64()) - return downloader.FullSync, td -} - -// startSync launches doSync in a new goroutine. -func (cs *chainSyncer) startSync(op *chainSyncOp) { - cs.doneCh = make(chan error, 1) - go func() { cs.doneCh <- cs.handler.doSync(op) }() -} - -// doSync synchronizes the local blockchain with a remote peer. -func (h *handler) doSync(op *chainSyncOp) error { - // Run the sync cycle, and disable snap sync if we're past the pivot block - err := h.downloader.LegacySync(op.peer.ID(), op.head, op.td, h.chain.Config().TerminalTotalDifficulty, op.mode) - if err != nil { - return err - } - h.enableSyncedFeatures() - - head := h.chain.CurrentBlock() - if head.Number.Uint64() > 0 { - // We've completed a sync cycle, notify all peers of new state. This path is - // essential in star-topology networks where a gateway node needs to notify - // all its out-of-date peers of the availability of a new block. This failure - // scenario will most often crop up in private and hackathon networks with - // degenerate connectivity, but it should be healthy for the mainnet too to - // more reliably update peers or the local TD state. - if block := h.chain.GetBlock(head.Hash(), head.Number.Uint64()); block != nil { - h.BroadcastBlock(block, false) - } - } - return nil -} diff --git a/eth/sync_test.go b/eth/sync_test.go index a31986730f06..7ede0a82c5d5 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -85,10 +85,11 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) { time.Sleep(250 * time.Millisecond) // Check that snap sync was disabled - op := peerToSyncOp(downloader.SnapSync, empty.handler.peers.peerWithHighestTD()) - if err := empty.handler.doSync(op); err != nil { + if err := empty.handler.downloader.BeaconSync(downloader.SnapSync, full.chain.CurrentBlock(), nil); err != nil { t.Fatal("sync failed:", err) } + empty.handler.enableSyncedFeatures() + if empty.handler.snapSync.Load() { t.Fatalf("snap sync not disabled after successful synchronisation") } diff --git a/params/config.go b/params/config.go index b24e782b8d96..439e88218924 100644 --- a/params/config.go +++ b/params/config.go @@ -361,6 +361,8 @@ type ChainConfig struct { // TerminalTotalDifficultyPassed is a flag specifying that the network already // passed the terminal total difficulty. Its purpose is to disable legacy sync // even without having seen the TTD locally (safer long term). + // + // TODO(karalabe): Drop this field eventually (always assuming PoS mode) TerminalTotalDifficultyPassed bool `json:"terminalTotalDifficultyPassed,omitempty"` // Various consensus engines From 66e1a6ef496e001abc7ae7433282251a557deb2c Mon Sep 17 00:00:00 2001 From: Devon Bear Date: Tue, 5 Mar 2024 09:15:02 -0500 Subject: [PATCH 60/91] go.mod: bump pebble db to official release (#29038) bump pebble --- ethdb/pebble/pebble.go | 4 +- go.mod | 13 ++-- go.sum | 148 ++++------------------------------------- 3 files changed, 21 insertions(+), 144 deletions(-) diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index af4686cf5b72..57689ab04b9b 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -589,8 +589,8 @@ func (b *batch) Reset() { func (b *batch) Replay(w ethdb.KeyValueWriter) error { reader := b.b.Reader() for { - kind, k, v, ok := reader.Next() - if !ok { + kind, k, v, ok, err := reader.Next() + if !ok || err != nil { break } // The (k,v) slices might be overwritten if the batch is reset/reused, diff --git a/go.mod b/go.mod index 48faa0a321d3..870ce76cb6e9 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.2.0 github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.79.0 - github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 + github.com/cockroachdb/pebble v1.1.0 github.com/consensys/gnark-crypto v0.12.1 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 github.com/crate-crypto/go-kzg-4844 v0.7.0 @@ -91,10 +91,9 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cockroachdb/errors v1.8.1 // indirect - github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect - github.com/cockroachdb/redact v1.0.8 // indirect - github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect + github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -102,11 +101,11 @@ require ( github.com/deepmap/oapi-codegen v1.6.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -140,7 +139,7 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.18.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 20a95d36877c..58006701c69b 100644 --- a/go.sum +++ b/go.sum @@ -31,7 +31,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= @@ -44,20 +43,14 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkM github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= -github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -65,7 +58,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= @@ -92,7 +84,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwF github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -119,30 +110,21 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.79.0 h1:ErwCYDjFCYppDJlDJ/5WhsSmzegAUe2+K9qgFyQDg3M= github.com/cloudflare/cloudflare-go v0.79.0/go.mod h1:gkHQf9xEubaQPEuerBuoinR9P8bf8a05Lq0X6WKy1Oc= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= -github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= -github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= -github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= -github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= -github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= -github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= +github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= @@ -162,9 +144,7 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/deepmap/oapi-codegen v1.6.0 h1:w/d1ntwh91XI0b/8ja7+u5SvA4IFfM0UNNLmiDR1gg0= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= @@ -174,45 +154,36 @@ github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 h1:qwcF+vdFrvPSEUDSX5R github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk= github.com/ferranbt/fastssz v0.1.2/go.mod h1:X5UPrE2u1UJjxHA8X54u04SBwdAQjG2sFtWs39YxyWs= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 h1:IZqZOB2fydHte3kUgxrzK5E1fW7RQGeDwE8F/ZZnUYc= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= -github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2GihuqhwdILrV+7GJel5lyPV3u1+PgzrWLc0TkE= github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -222,7 +193,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -231,20 +201,13 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -280,7 +243,6 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= -github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -295,8 +257,6 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -319,9 +279,7 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= @@ -334,10 +292,8 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6wn4Ej8vjuVGxeHdan+bRb2ebyv4= github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -347,21 +303,14 @@ github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXei github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb-client-go/v2 v2.4.0 h1:HGBfZYStlx3Kqvsv1h2pJixbCl/jhnFtxpKFAv9Tu5k= github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c h1:qSHzRbhzK8RdXOsAdfDgO49TtqC1oZ+acxPrkfTxcCs= github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= -github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= -github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= -github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY= @@ -377,30 +326,17 @@ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karalabe/hid v1.0.1-0.20240209121748-d3b59fe37df8 h1:IMoiklsIksD2ii43zKCybVU6jLNzpSl3bT31+5mUjgg= github.com/karalabe/hid v1.0.1-0.20240209121748-d3b59fe37df8/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= -github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= -github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= -github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= -github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -418,11 +354,9 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= @@ -432,7 +366,6 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -444,17 +377,11 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= -github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= -github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= @@ -467,23 +394,18 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= -github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= -github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -491,11 +413,9 @@ github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -539,26 +459,13 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -580,26 +487,13 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -613,11 +507,9 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -664,11 +556,9 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -677,7 +567,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -725,7 +614,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -735,7 +623,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -808,15 +695,11 @@ golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -883,7 +766,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -913,7 +795,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -938,8 +819,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -948,9 +829,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= From 588c5480fd1f355a39d3f52a5507ab9d0da334c9 Mon Sep 17 00:00:00 2001 From: Tom <45168162+tomdever@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:23:35 +0800 Subject: [PATCH 61/91] internal/ethapi: delete needless error check (#29127) --- internal/ethapi/api_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 8ffa638a6b5c..1d0383daad03 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -1244,7 +1244,7 @@ func TestFillBlobTransaction(t *testing.T) { if len(tc.err) > 0 { if err == nil { t.Fatalf("missing error. want: %s", tc.err) - } else if err != nil && err.Error() != tc.err { + } else if err.Error() != tc.err { t.Fatalf("error mismatch. want: %s, have: %s", tc.err, err.Error()) } return From 899bb88a4ba19af2d8fe4874561a9d55355acf48 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Wed, 6 Mar 2024 10:32:17 +0100 Subject: [PATCH 62/91] accounts/usbwallet: revert #28945 (#29175) --- accounts/usbwallet/hub.go | 8 ++++---- accounts/usbwallet/wallet.go | 6 +++--- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/accounts/usbwallet/hub.go b/accounts/usbwallet/hub.go index e22dffe9718e..e67942dbc107 100644 --- a/accounts/usbwallet/hub.go +++ b/accounts/usbwallet/hub.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" - "github.com/karalabe/hid" + "github.com/karalabe/usb" ) // LedgerScheme is the protocol scheme prefixing account and wallet URLs. @@ -109,7 +109,7 @@ func NewTrezorHubWithWebUSB() (*Hub, error) { // newHub creates a new hardware wallet manager for generic USB devices. func newHub(scheme string, vendorID uint16, productIDs []uint16, usageID uint16, endpointID int, makeDriver func(log.Logger) driver) (*Hub, error) { - if !hid.Supported() { + if !usb.Supported() { return nil, errors.New("unsupported platform") } hub := &Hub{ @@ -155,7 +155,7 @@ func (hub *Hub) refreshWallets() { return } // Retrieve the current list of USB wallet devices - var devices []hid.DeviceInfo + var devices []usb.DeviceInfo if runtime.GOOS == "linux" { // hidapi on Linux opens the device during enumeration to retrieve some infos, @@ -170,7 +170,7 @@ func (hub *Hub) refreshWallets() { return } } - infos, err := hid.Enumerate(hub.vendorID, 0) + infos, err := usb.Enumerate(hub.vendorID, 0) if err != nil { failcount := hub.enumFails.Add(1) if runtime.GOOS == "linux" { diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 0fd0415a9ef8..69083dc8939d 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/karalabe/hid" + "github.com/karalabe/usb" ) // Maximum time between wallet health checks to detect USB unplugs. @@ -79,8 +79,8 @@ type wallet struct { driver driver // Hardware implementation of the low level device operations url *accounts.URL // Textual URL uniquely identifying this wallet - info hid.DeviceInfo // Known USB device infos about the wallet - device hid.Device // USB device advertising itself as a hardware wallet + info usb.DeviceInfo // Known USB device infos about the wallet + device usb.Device // USB device advertising itself as a hardware wallet accounts []accounts.Account // List of derive accounts pinned on the hardware wallet paths map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations diff --git a/go.mod b/go.mod index 870ce76cb6e9..42114e115a55 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 github.com/julienschmidt/httprouter v1.3.0 - github.com/karalabe/hid v1.0.1-0.20240209121748-d3b59fe37df8 + github.com/karalabe/usb v0.0.2 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.17 diff --git a/go.sum b/go.sum index 58006701c69b..e9f62bbfd75d 100644 --- a/go.sum +++ b/go.sum @@ -329,8 +329,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karalabe/hid v1.0.1-0.20240209121748-d3b59fe37df8 h1:IMoiklsIksD2ii43zKCybVU6jLNzpSl3bT31+5mUjgg= -github.com/karalabe/hid v1.0.1-0.20240209121748-d3b59fe37df8/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= +github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= From a000acb61114c2a3a74c065f2e61b4d6bca3ae46 Mon Sep 17 00:00:00 2001 From: Andrei Kostakov Date: Wed, 6 Mar 2024 11:53:12 +0200 Subject: [PATCH 63/91] rpc: add more test cases for arg types (#29006) --- rpc/types_test.go | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/rpc/types_test.go b/rpc/types_test.go index 617f441d9166..2fa74f9899bb 100644 --- a/rpc/types_test.go +++ b/rpc/types_test.go @@ -45,9 +45,11 @@ func TestBlockNumberJSONUnmarshal(t *testing.T) { 11: {`"pending"`, false, PendingBlockNumber}, 12: {`"latest"`, false, LatestBlockNumber}, 13: {`"earliest"`, false, EarliestBlockNumber}, - 14: {`someString`, true, BlockNumber(0)}, - 15: {`""`, true, BlockNumber(0)}, - 16: {``, true, BlockNumber(0)}, + 14: {`"safe"`, false, SafeBlockNumber}, + 15: {`"finalized"`, false, FinalizedBlockNumber}, + 16: {`someString`, true, BlockNumber(0)}, + 17: {`""`, true, BlockNumber(0)}, + 18: {``, true, BlockNumber(0)}, } for i, test := range tests { @@ -87,18 +89,22 @@ func TestBlockNumberOrHash_UnmarshalJSON(t *testing.T) { 11: {`"pending"`, false, BlockNumberOrHashWithNumber(PendingBlockNumber)}, 12: {`"latest"`, false, BlockNumberOrHashWithNumber(LatestBlockNumber)}, 13: {`"earliest"`, false, BlockNumberOrHashWithNumber(EarliestBlockNumber)}, - 14: {`someString`, true, BlockNumberOrHash{}}, - 15: {`""`, true, BlockNumberOrHash{}}, - 16: {``, true, BlockNumberOrHash{}}, - 17: {`"0x0000000000000000000000000000000000000000000000000000000000000000"`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, - 18: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, - 19: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":false}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, - 20: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":true}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), true)}, - 21: {`{"blockNumber":"0x1"}`, false, BlockNumberOrHashWithNumber(1)}, - 22: {`{"blockNumber":"pending"}`, false, BlockNumberOrHashWithNumber(PendingBlockNumber)}, - 23: {`{"blockNumber":"latest"}`, false, BlockNumberOrHashWithNumber(LatestBlockNumber)}, - 24: {`{"blockNumber":"earliest"}`, false, BlockNumberOrHashWithNumber(EarliestBlockNumber)}, - 25: {`{"blockNumber":"0x1", "blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, true, BlockNumberOrHash{}}, + 14: {`"safe"`, false, BlockNumberOrHashWithNumber(SafeBlockNumber)}, + 15: {`"finalized"`, false, BlockNumberOrHashWithNumber(FinalizedBlockNumber)}, + 16: {`someString`, true, BlockNumberOrHash{}}, + 17: {`""`, true, BlockNumberOrHash{}}, + 18: {``, true, BlockNumberOrHash{}}, + 19: {`"0x0000000000000000000000000000000000000000000000000000000000000000"`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, + 20: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, + 21: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":false}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, + 22: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":true}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), true)}, + 23: {`{"blockNumber":"0x1"}`, false, BlockNumberOrHashWithNumber(1)}, + 24: {`{"blockNumber":"pending"}`, false, BlockNumberOrHashWithNumber(PendingBlockNumber)}, + 25: {`{"blockNumber":"latest"}`, false, BlockNumberOrHashWithNumber(LatestBlockNumber)}, + 26: {`{"blockNumber":"earliest"}`, false, BlockNumberOrHashWithNumber(EarliestBlockNumber)}, + 27: {`{"blockNumber":"safe"}`, false, BlockNumberOrHashWithNumber(SafeBlockNumber)}, + 28: {`{"blockNumber":"finalized"}`, false, BlockNumberOrHashWithNumber(FinalizedBlockNumber)}, + 29: {`{"blockNumber":"0x1", "blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, true, BlockNumberOrHash{}}, } for i, test := range tests { @@ -133,6 +139,8 @@ func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { {"pending", int64(PendingBlockNumber)}, {"latest", int64(LatestBlockNumber)}, {"earliest", int64(EarliestBlockNumber)}, + {"safe", int64(SafeBlockNumber)}, + {"finalized", int64(FinalizedBlockNumber)}, } for _, test := range tests { test := test @@ -160,6 +168,8 @@ func TestBlockNumberOrHash_StringAndUnmarshal(t *testing.T) { BlockNumberOrHashWithNumber(PendingBlockNumber), BlockNumberOrHashWithNumber(LatestBlockNumber), BlockNumberOrHashWithNumber(EarliestBlockNumber), + BlockNumberOrHashWithNumber(SafeBlockNumber), + BlockNumberOrHashWithNumber(FinalizedBlockNumber), BlockNumberOrHashWithNumber(32), BlockNumberOrHashWithHash(common.Hash{0xaa}, false), } From e73f55365c458c5185a493935b65dd96bacf6933 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Wed, 6 Mar 2024 11:31:50 +0100 Subject: [PATCH 64/91] accounts/usbwallet: update hid library (#29176) --- accounts/usbwallet/hub.go | 8 ++++---- accounts/usbwallet/wallet.go | 6 +++--- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/accounts/usbwallet/hub.go b/accounts/usbwallet/hub.go index e67942dbc107..e22dffe9718e 100644 --- a/accounts/usbwallet/hub.go +++ b/accounts/usbwallet/hub.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" - "github.com/karalabe/usb" + "github.com/karalabe/hid" ) // LedgerScheme is the protocol scheme prefixing account and wallet URLs. @@ -109,7 +109,7 @@ func NewTrezorHubWithWebUSB() (*Hub, error) { // newHub creates a new hardware wallet manager for generic USB devices. func newHub(scheme string, vendorID uint16, productIDs []uint16, usageID uint16, endpointID int, makeDriver func(log.Logger) driver) (*Hub, error) { - if !usb.Supported() { + if !hid.Supported() { return nil, errors.New("unsupported platform") } hub := &Hub{ @@ -155,7 +155,7 @@ func (hub *Hub) refreshWallets() { return } // Retrieve the current list of USB wallet devices - var devices []usb.DeviceInfo + var devices []hid.DeviceInfo if runtime.GOOS == "linux" { // hidapi on Linux opens the device during enumeration to retrieve some infos, @@ -170,7 +170,7 @@ func (hub *Hub) refreshWallets() { return } } - infos, err := usb.Enumerate(hub.vendorID, 0) + infos, err := hid.Enumerate(hub.vendorID, 0) if err != nil { failcount := hub.enumFails.Add(1) if runtime.GOOS == "linux" { diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 69083dc8939d..0fd0415a9ef8 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/karalabe/usb" + "github.com/karalabe/hid" ) // Maximum time between wallet health checks to detect USB unplugs. @@ -79,8 +79,8 @@ type wallet struct { driver driver // Hardware implementation of the low level device operations url *accounts.URL // Textual URL uniquely identifying this wallet - info usb.DeviceInfo // Known USB device infos about the wallet - device usb.Device // USB device advertising itself as a hardware wallet + info hid.DeviceInfo // Known USB device infos about the wallet + device hid.Device // USB device advertising itself as a hardware wallet accounts []accounts.Account // List of derive accounts pinned on the hardware wallet paths map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations diff --git a/go.mod b/go.mod index 42114e115a55..6591bee62ff1 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 github.com/julienschmidt/httprouter v1.3.0 - github.com/karalabe/usb v0.0.2 + github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.17 diff --git a/go.sum b/go.sum index e9f62bbfd75d..cc74e15cb4b9 100644 --- a/go.sum +++ b/go.sum @@ -329,8 +329,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= -github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 h1:msKODTL1m0wigztaqILOtla9HeW1ciscYG4xjLtvk5I= +github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= From a90fe84971183aa0b6c40d71c6586ae3f2eda4c8 Mon Sep 17 00:00:00 2001 From: Undefinedor Date: Wed, 6 Mar 2024 18:55:44 +0800 Subject: [PATCH 65/91] accounts: remove deprecated function NewPlaintextKeyStore (#29171) --- accounts/keystore/account_cache_test.go | 2 +- accounts/keystore/keystore.go | 9 ------- accounts/keystore/keystore_test.go | 34 +++++++++++-------------- 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index 48a238048fec..bb92cc2adca5 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -86,7 +86,7 @@ func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error { func TestWatchNewFile(t *testing.T) { t.Parallel() - dir, ks := tmpKeyStore(t, false) + dir, ks := tmpKeyStore(t) // Ensure the watcher is started before adding any files. ks.Accounts() diff --git a/accounts/keystore/keystore.go b/accounts/keystore/keystore.go index 0ffcf376a5fd..e62a8eb25738 100644 --- a/accounts/keystore/keystore.go +++ b/accounts/keystore/keystore.go @@ -87,15 +87,6 @@ func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore { return ks } -// NewPlaintextKeyStore creates a keystore for the given directory. -// Deprecated: Use NewKeyStore. -func NewPlaintextKeyStore(keydir string) *KeyStore { - keydir, _ = filepath.Abs(keydir) - ks := &KeyStore{storage: &keyStorePlain{keydir}} - ks.init(keydir) - return ks -} - func (ks *KeyStore) init(keydir string) { // Lock the mutex since the account cache might call back with events ks.mu.Lock() diff --git a/accounts/keystore/keystore_test.go b/accounts/keystore/keystore_test.go index c9a23eddd6ca..c871392b82ec 100644 --- a/accounts/keystore/keystore_test.go +++ b/accounts/keystore/keystore_test.go @@ -37,7 +37,7 @@ var testSigData = make([]byte, 32) func TestKeyStore(t *testing.T) { t.Parallel() - dir, ks := tmpKeyStore(t, true) + dir, ks := tmpKeyStore(t) a, err := ks.NewAccount("foo") if err != nil { @@ -72,7 +72,7 @@ func TestKeyStore(t *testing.T) { func TestSign(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) pass := "" // not used but required by API a1, err := ks.NewAccount(pass) @@ -89,7 +89,7 @@ func TestSign(t *testing.T) { func TestSignWithPassphrase(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) pass := "passwd" acc, err := ks.NewAccount(pass) @@ -117,7 +117,7 @@ func TestSignWithPassphrase(t *testing.T) { func TestTimedUnlock(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) pass := "foo" a1, err := ks.NewAccount(pass) @@ -152,7 +152,7 @@ func TestTimedUnlock(t *testing.T) { func TestOverrideUnlock(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) pass := "foo" a1, err := ks.NewAccount(pass) @@ -193,7 +193,7 @@ func TestOverrideUnlock(t *testing.T) { // This test should fail under -race if signing races the expiration goroutine. func TestSignRace(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) // Create a test account. a1, err := ks.NewAccount("") @@ -238,7 +238,7 @@ func waitForKsUpdating(t *testing.T, ks *KeyStore, wantStatus bool, maxTime time func TestWalletNotifierLifecycle(t *testing.T) { t.Parallel() // Create a temporary keystore to test with - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) // Ensure that the notification updater is not running yet time.Sleep(250 * time.Millisecond) @@ -284,7 +284,7 @@ type walletEvent struct { // or deleted from the keystore. func TestWalletNotifications(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) // Subscribe to the wallet feed and collect events. var ( @@ -346,7 +346,7 @@ func TestWalletNotifications(t *testing.T) { // TestImportExport tests the import functionality of a keystore. func TestImportECDSA(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) key, err := crypto.GenerateKey() if err != nil { t.Fatalf("failed to generate key: %v", key) @@ -365,7 +365,7 @@ func TestImportECDSA(t *testing.T) { // TestImportECDSA tests the import and export functionality of a keystore. func TestImportExport(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) acc, err := ks.NewAccount("old") if err != nil { t.Fatalf("failed to create account: %v", acc) @@ -374,7 +374,7 @@ func TestImportExport(t *testing.T) { if err != nil { t.Fatalf("failed to export account: %v", acc) } - _, ks2 := tmpKeyStore(t, true) + _, ks2 := tmpKeyStore(t) if _, err = ks2.Import(json, "old", "old"); err == nil { t.Errorf("importing with invalid password succeeded") } @@ -394,7 +394,7 @@ func TestImportExport(t *testing.T) { // This test should fail under -race if importing races. func TestImportRace(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) acc, err := ks.NewAccount("old") if err != nil { t.Fatalf("failed to create account: %v", acc) @@ -403,7 +403,7 @@ func TestImportRace(t *testing.T) { if err != nil { t.Fatalf("failed to export account: %v", acc) } - _, ks2 := tmpKeyStore(t, true) + _, ks2 := tmpKeyStore(t) var atom atomic.Uint32 var wg sync.WaitGroup wg.Add(2) @@ -457,11 +457,7 @@ func checkEvents(t *testing.T, want []walletEvent, have []walletEvent) { } } -func tmpKeyStore(t *testing.T, encrypted bool) (string, *KeyStore) { +func tmpKeyStore(t *testing.T) (string, *KeyStore) { d := t.TempDir() - newKs := NewPlaintextKeyStore - if encrypted { - newKs = func(kd string) *KeyStore { return NewKeyStore(kd, veryLightScryptN, veryLightScryptP) } - } - return d, newKs(d) + return d, NewKeyStore(d, veryLightScryptN, veryLightScryptP) } From 6e379b6fc776668c9a7db6d5b014d0dd89d7118d Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 6 Mar 2024 20:36:12 +0800 Subject: [PATCH 66/91] eth/tracers: prestate tracer add blob fee (#29168) * eth/tracers: prestate balance add blob fee Signed-off-by: jsvisa * eth/tracers: prestate test support blob tx Signed-off-by: jsvisa * eth/tracers: add prestate blob tx test Signed-off-by: jsvisa --------- Signed-off-by: jsvisa --- .../internal/tracetest/prestate_test.go | 6 ++ .../testdata/prestate_tracer/blob_tx.json | 63 +++++++++++++++++++ eth/tracers/native/prestate.go | 7 +++ 3 files changed, 76 insertions(+) create mode 100644 eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index 8a60123dc2c6..38097ff334b2 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" @@ -107,6 +108,11 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { ) defer state.Close() + if test.Genesis.ExcessBlobGas != nil && test.Genesis.BlobGasUsed != nil { + excessBlobGas := eip4844.CalcExcessBlobGas(*test.Genesis.ExcessBlobGas, *test.Genesis.BlobGasUsed) + context.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas) + } + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json new file mode 100644 index 000000000000..315481aff536 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json @@ -0,0 +1,63 @@ +{ + "genesis": { + "baseFeePerGas": "7", + "blobGasUsed": "0", + "difficulty": "0", + "excessBlobGas": "36306944", + "extraData": "0xd983010e00846765746888676f312e32312e308664617277696e", + "gasLimit": "15639172", + "hash": "0xc682259fda061bb9ce8ccb491d5b2d436cb73daf04e1025dd116d045ce4ad28c", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0xae1a5ba939a4c9ac38aabeff361169fb55a6fc2c9511457e0be6eff9514faec0", + "nonce": "0x0000000000000000", + "number": "315", + "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "stateRoot": "0x577f42ab21ccfd946511c57869ace0bdf7c217c36f02b7cd3459df0ed1cffc1a", + "timestamp": "1709626771", + "totalDifficulty": "1", + "withdrawals": [], + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "alloc": { + "0x0000000000000000000000000000000000000000": { + "balance": "0x272e0528" + }, + "0x0c2c51a0990aee1d73c1228de158688341557508": { + "balance": "0xde0b6b3a7640000" + } + }, + "config": { + "chainId": 1337, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "shanghaiTime": 0, + "cancunTime": 0, + "terminalTotalDifficulty": 0, + "terminalTotalDifficultyPassed": true + } + }, + "context": { + "number": "316", + "difficulty": "0", + "timestamp": "1709626785", + "gasLimit": "15654443", + "miner": "0x0000000000000000000000000000000000000000" + }, + "input": "0x03f8b1820539806485174876e800825208940c2c51a0990aee1d73c1228de1586883415575088080c083020000f842a00100c9fbdf97f747e85847b4f3fff408f89c26842f77c882858bf2c89923849aa00138e3896f3c27f2389147507f8bcec52028b0efca6ee842ed83c9158873943880a0dbac3f97a532c9b00e6239b29036245a5bfbb96940b9d848634661abee98b945a03eec8525f261c2e79798f7b45a5d6ccaefa24576d53ba5023e919b86841c0675", + "result": { + "0x0000000000000000000000000000000000000000": { "balance": "0x272e0528" }, + "0x0c2c51a0990aee1d73c1228de158688341557508": { + "balance": "0xde0b6b3a7640000" + } + } +} diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index 0d57f62caf20..b86c5c461c7d 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" ) //go:generate go run github.com/fjl/gencodec -type account -field-override accountMarshaling -out gen_account_json.go @@ -112,6 +113,12 @@ func (t *prestateTracer) CaptureStart(env *vm.EVM, from common.Address, to commo gasPrice := env.TxContext.GasPrice consumedGas := new(big.Int).Mul(gasPrice, new(big.Int).SetUint64(t.gasLimit)) fromBal.Add(fromBal, new(big.Int).Add(value, consumedGas)) + + // Add blob fee to the sender's balance. + if env.Context.BlobBaseFee != nil && len(env.TxContext.BlobHashes) > 0 { + blobGas := uint64(params.BlobTxBlobGasPerBlob * len(env.TxContext.BlobHashes)) + fromBal.Add(fromBal, new(big.Int).Mul(env.Context.BlobBaseFee, new(big.Int).SetUint64(blobGas))) + } t.pre[from].Balance = fromBal t.pre[from].Nonce-- From d8e0807da22eb922539d15b0d5d01ccdd58b1267 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 6 Mar 2024 13:45:03 +0100 Subject: [PATCH 67/91] miner: refactor the miner, make the pending block on demand (#28623) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * miner: untangle miner * miner: use common.hash instead of *types.header * cmd/geth: deprecate --mine * eth: get rid of most miner api * console: get rid of coinbase in welcome message * miner/stress: get rid of the miner stress test * eth: get rid of miner.setEtherbase * ethstats: remove miner and hashrate flags * ethstats: remove miner and hashrate flags * cmd: rename pendingBlockProducer to miner.pending.feeRecipient flag * miner: use pendingFeeRecipient instead of etherbase * miner: add mutex to protect the pending block * miner: add mutex to protect the pending block * eth: get rid of etherbase mentions * miner: no need to lock the coinbase * eth, miner: fix linter --------- Co-authored-by: Martin Holst Swende Co-authored-by: Péter Szilágyi --- cmd/geth/consolecmd_test.go | 3 - cmd/geth/main.go | 26 +- cmd/utils/flags.go | 43 +- cmd/utils/flags_legacy.go | 20 + consensus/consensus.go | 8 - console/console.go | 3 - console/console_test.go | 5 +- eth/api.go | 52 - eth/api_backend.go | 23 +- eth/api_debug.go | 4 +- eth/api_miner.go | 28 - eth/backend.go | 152 +-- eth/catalyst/api_test.go | 5 +- eth/catalyst/simulated_beacon_test.go | 3 +- eth/filters/filter.go | 2 +- eth/filters/filter_system.go | 78 +- eth/filters/filter_system_test.go | 42 +- eth/filters/filter_test.go | 7 +- eth/gasprice/feehistory.go | 2 +- eth/gasprice/gasprice.go | 3 +- eth/gasprice/gasprice_test.go | 8 +- ethclient/ethclient_test.go | 23 +- ethstats/ethstats.go | 25 +- internal/ethapi/api_test.go | 5 +- internal/ethapi/backend.go | 3 +- internal/ethapi/transaction_args_test.go | 5 +- internal/web3ext/web3ext.go | 23 - miner/miner.go | 256 ++--- miner/miner_test.go | 204 +--- miner/payload_building.go | 9 +- miner/payload_building_test.go | 119 ++- miner/pending.go | 67 ++ miner/stress/clique/main.go | 223 ----- miner/worker.go | 1097 +++------------------- miner/worker_test.go | 510 ---------- 35 files changed, 598 insertions(+), 2488 deletions(-) delete mode 100644 eth/api.go create mode 100644 miner/pending.go delete mode 100644 miner/stress/clique/main.go delete mode 100644 miner/worker_test.go diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index ef6ef5f28836..4d6220641703 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -71,7 +71,6 @@ func TestConsoleWelcome(t *testing.T) { Welcome to the Geth JavaScript console! instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}} -coinbase: {{.Etherbase}} at block: 0 ({{niltime}}) datadir: {{.Datadir}} modules: {{apis}} @@ -131,7 +130,6 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) { attach.SetTemplateFunc("goarch", func() string { return runtime.GOARCH }) attach.SetTemplateFunc("gover", runtime.Version) attach.SetTemplateFunc("gethver", func() string { return params.VersionWithCommit("", "") }) - attach.SetTemplateFunc("etherbase", func() string { return geth.Etherbase }) attach.SetTemplateFunc("niltime", func() string { return time.Unix(1548854791, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)") }) @@ -144,7 +142,6 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) { Welcome to the Geth JavaScript console! instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}} -coinbase: {{etherbase}} at block: 0 ({{niltime}}){{if ipc}} datadir: {{datadir}}{{end}} modules: {{apis}} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 2f7d37fdd7e7..9a88e9f2e8b4 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console/prompt" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/internal/debug" @@ -116,13 +115,14 @@ var ( utils.DiscoveryPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, - utils.MiningEnabledFlag, + utils.MiningEnabledFlag, // deprecated utils.MinerGasLimitFlag, utils.MinerGasPriceFlag, - utils.MinerEtherbaseFlag, + utils.MinerEtherbaseFlag, // deprecated utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, - utils.MinerNewPayloadTimeout, + utils.MinerPendingFeeRecipientFlag, + utils.MinerNewPayloadTimeoutFlag, // deprecated utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV4Flag, @@ -421,24 +421,6 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon } }() } - - // Start auxiliary services if enabled - if ctx.Bool(utils.MiningEnabledFlag.Name) { - // Mining only makes sense if a full Ethereum node is running - if ctx.String(utils.SyncModeFlag.Name) == "light" { - utils.Fatalf("Light clients do not support mining") - } - ethBackend, ok := backend.(*eth.EthAPIBackend) - if !ok { - utils.Fatalf("Ethereum service not running") - } - // Set the gas price to the limits from the CLI and start mining - gasprice := flags.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) - ethBackend.TxPool().SetGasTip(gasprice) - if err := ethBackend.StartMining(); err != nil { - utils.Fatalf("Failed to start mining: %v", err) - } - } } // unlockAccounts unlocks any account specifically requested. diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 82af26ff9659..fad567cd55d2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -425,11 +425,6 @@ var ( } // Miner settings - MiningEnabledFlag = &cli.BoolFlag{ - Name: "mine", - Usage: "Enable mining", - Category: flags.MinerCategory, - } MinerGasLimitFlag = &cli.Uint64Flag{ Name: "miner.gaslimit", Usage: "Target gas ceiling for mined blocks", @@ -442,11 +437,6 @@ var ( Value: ethconfig.Defaults.Miner.GasPrice, Category: flags.MinerCategory, } - MinerEtherbaseFlag = &cli.StringFlag{ - Name: "miner.etherbase", - Usage: "0x prefixed public address for block mining rewards", - Category: flags.MinerCategory, - } MinerExtraDataFlag = &cli.StringFlag{ Name: "miner.extradata", Usage: "Block extra data set by the miner (default = client version)", @@ -458,10 +448,9 @@ var ( Value: ethconfig.Defaults.Miner.Recommit, Category: flags.MinerCategory, } - MinerNewPayloadTimeout = &cli.DurationFlag{ - Name: "miner.newpayload-timeout", - Usage: "Specify the maximum time allowance for creating a new payload", - Value: ethconfig.Defaults.Miner.NewPayloadTimeout, + MinerPendingFeeRecipientFlag = &cli.StringFlag{ + Name: "miner.pending.feeRecipient", + Usage: "0x prefixed public address for the pending block producer (not used for actual block production)", Category: flags.MinerCategory, } @@ -1268,19 +1257,23 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error // setEtherbase retrieves the etherbase from the directly specified command line flags. func setEtherbase(ctx *cli.Context, cfg *ethconfig.Config) { - if !ctx.IsSet(MinerEtherbaseFlag.Name) { + if ctx.IsSet(MinerEtherbaseFlag.Name) { + log.Warn("Option --miner.etherbase is deprecated as the etherbase is set by the consensus client post-merge") return } - addr := ctx.String(MinerEtherbaseFlag.Name) + if !ctx.IsSet(MinerPendingFeeRecipientFlag.Name) { + return + } + addr := ctx.String(MinerPendingFeeRecipientFlag.Name) if strings.HasPrefix(addr, "0x") || strings.HasPrefix(addr, "0X") { addr = addr[2:] } b, err := hex.DecodeString(addr) if err != nil || len(b) != common.AddressLength { - Fatalf("-%s: invalid etherbase address %q", MinerEtherbaseFlag.Name, addr) + Fatalf("-%s: invalid pending block producer address %q", MinerPendingFeeRecipientFlag.Name, addr) return } - cfg.Miner.Etherbase = common.BytesToAddress(b) + cfg.Miner.PendingFeeRecipient = common.BytesToAddress(b) } // MakePasswordList reads password lines from the file specified by the global --password flag. @@ -1496,6 +1489,9 @@ func setTxPool(ctx *cli.Context, cfg *legacypool.Config) { } func setMiner(ctx *cli.Context, cfg *miner.Config) { + if ctx.Bool(MiningEnabledFlag.Name) { + log.Warn("The flag --mine is deprecated and will be removed") + } if ctx.IsSet(MinerExtraDataFlag.Name) { cfg.ExtraData = []byte(ctx.String(MinerExtraDataFlag.Name)) } @@ -1508,8 +1504,9 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.IsSet(MinerRecommitIntervalFlag.Name) { cfg.Recommit = ctx.Duration(MinerRecommitIntervalFlag.Name) } - if ctx.IsSet(MinerNewPayloadTimeout.Name) { - cfg.NewPayloadTimeout = ctx.Duration(MinerNewPayloadTimeout.Name) + if ctx.IsSet(MinerNewPayloadTimeoutFlag.Name) { + log.Warn("The flag --miner.newpayload-timeout is deprecated and will be removed, please use --miner.recommit") + cfg.Recommit = ctx.Duration(MinerNewPayloadTimeoutFlag.Name) } } @@ -1786,8 +1783,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // Figure out the dev account address. // setEtherbase has been called above, configuring the miner address from command line flags. - if cfg.Miner.Etherbase != (common.Address{}) { - developer = accounts.Account{Address: cfg.Miner.Etherbase} + if cfg.Miner.PendingFeeRecipient != (common.Address{}) { + developer = accounts.Account{Address: cfg.Miner.PendingFeeRecipient} } else if accs := ks.Accounts(); len(accs) > 0 { developer = ks.Accounts()[0] } else { @@ -1798,7 +1795,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } // Make sure the address is configured as fee recipient, otherwise // the miner will fail to start. - cfg.Miner.Etherbase = developer.Address + cfg.Miner.PendingFeeRecipient = developer.Address if err := ks.Unlock(developer, passphrase); err != nil { Fatalf("Failed to unlock developer account: %v", err) diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 243abd831105..49321053c672 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -47,6 +47,9 @@ var DeprecatedFlags = []cli.Flag{ LightNoSyncServeFlag, LogBacktraceAtFlag, LogDebugFlag, + MinerNewPayloadTimeoutFlag, + MinerEtherbaseFlag, + MiningEnabledFlag, } var ( @@ -132,6 +135,23 @@ var ( Usage: "Prepends log messages with call-site location (deprecated)", Category: flags.DeprecatedCategory, } + // Deprecated February 2024 + MinerNewPayloadTimeoutFlag = &cli.DurationFlag{ + Name: "miner.newpayload-timeout", + Usage: "Specify the maximum time allowance for creating a new payload", + Value: ethconfig.Defaults.Miner.Recommit, + Category: flags.MinerCategory, + } + MinerEtherbaseFlag = &cli.StringFlag{ + Name: "miner.etherbase", + Usage: "0x prefixed public address for block mining rewards", + Category: flags.MinerCategory, + } + MiningEnabledFlag = &cli.BoolFlag{ + Name: "mine", + Usage: "Enable mining", + Category: flags.MinerCategory, + } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. diff --git a/consensus/consensus.go b/consensus/consensus.go index 3a2c2d222916..5cc052cb0fea 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -119,11 +119,3 @@ type Engine interface { // Close terminates any background threads maintained by the consensus engine. Close() error } - -// PoW is a consensus engine based on proof-of-work. -type PoW interface { - Engine - - // Hashrate returns the current mining hashrate of a PoW consensus engine. - Hashrate() float64 -} diff --git a/console/console.go b/console/console.go index cdee53684ecf..5acb4cdccb5b 100644 --- a/console/console.go +++ b/console/console.go @@ -325,9 +325,6 @@ func (c *Console) Welcome() { // Print some generic Geth metadata if res, err := c.jsre.Run(` var message = "instance: " + web3.version.node + "\n"; - try { - message += "coinbase: " + eth.coinbase + "\n"; - } catch (err) {} message += "at block: " + eth.blockNumber + " (" + new Date(1000 * eth.getBlock(eth.blockNumber).timestamp) + ")\n"; try { message += " datadir: " + admin.datadir + "\n"; diff --git a/console/console_test.go b/console/console_test.go index a13be6a99ded..4c30c1b49cc6 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -96,7 +96,7 @@ func newTester(t *testing.T, confOverride func(*ethconfig.Config)) *tester { ethConf := ðconfig.Config{ Genesis: core.DeveloperGenesisBlock(11_500_000, nil), Miner: miner.Config{ - Etherbase: common.HexToAddress(testAddress), + PendingFeeRecipient: common.HexToAddress(testAddress), }, } if confOverride != nil { @@ -167,9 +167,6 @@ func TestWelcome(t *testing.T) { if want := fmt.Sprintf("instance: %s", testInstance); !strings.Contains(output, want) { t.Fatalf("console output missing instance: have\n%s\nwant also %s", output, want) } - if want := fmt.Sprintf("coinbase: %s", testAddress); !strings.Contains(output, want) { - t.Fatalf("console output missing coinbase: have\n%s\nwant also %s", output, want) - } if want := "at block: 0"; !strings.Contains(output, want) { t.Fatalf("console output missing sync status: have\n%s\nwant also %s", output, want) } diff --git a/eth/api.go b/eth/api.go deleted file mode 100644 index 44e934fd040b..000000000000 --- a/eth/api.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" -) - -// EthereumAPI provides an API to access Ethereum full node-related information. -type EthereumAPI struct { - e *Ethereum -} - -// NewEthereumAPI creates a new Ethereum protocol API for full nodes. -func NewEthereumAPI(e *Ethereum) *EthereumAPI { - return &EthereumAPI{e} -} - -// Etherbase is the address that mining rewards will be sent to. -func (api *EthereumAPI) Etherbase() (common.Address, error) { - return api.e.Etherbase() -} - -// Coinbase is the address that mining rewards will be sent to (alias for Etherbase). -func (api *EthereumAPI) Coinbase() (common.Address, error) { - return api.Etherbase() -} - -// Hashrate returns the POW hashrate. -func (api *EthereumAPI) Hashrate() hexutil.Uint64 { - return hexutil.Uint64(api.e.Miner().Hashrate()) -} - -// Mining returns an indication if this node is currently mining. -func (api *EthereumAPI) Mining() bool { - return api.e.IsMining() -} diff --git a/eth/api_backend.go b/eth/api_backend.go index 65adccd8518c..48c46447c5a0 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -37,7 +37,6 @@ import ( "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -67,7 +66,7 @@ func (b *EthAPIBackend) SetHead(number uint64) { func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { // Pending block is only known by the miner if number == rpc.PendingBlockNumber { - block := b.eth.miner.PendingBlock() + block, _, _ := b.eth.miner.Pending() if block == nil { return nil, errors.New("pending block is not available") } @@ -118,7 +117,7 @@ func (b *EthAPIBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*ty func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { // Pending block is only known by the miner if number == rpc.PendingBlockNumber { - block := b.eth.miner.PendingBlock() + block, _, _ := b.eth.miner.Pending() if block == nil { return nil, errors.New("pending block is not available") } @@ -182,14 +181,14 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r return nil, errors.New("invalid arguments; neither block nor hash specified") } -func (b *EthAPIBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { - return b.eth.miner.PendingBlockAndReceipts() +func (b *EthAPIBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { + return b.eth.miner.Pending() } func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { // Pending state is only known by the miner if number == rpc.PendingBlockNumber { - block, state := b.eth.miner.Pending() + block, _, state := b.eth.miner.Pending() if block == nil || state == nil { return nil, nil, errors.New("pending state is not available") } @@ -267,10 +266,6 @@ func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEven return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch) } -func (b *EthAPIBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - return b.eth.miner.SubscribePendingLogs(ch) -} - func (b *EthAPIBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainEvent(ch) } @@ -421,14 +416,6 @@ func (b *EthAPIBackend) CurrentHeader() *types.Header { return b.eth.blockchain.CurrentHeader() } -func (b *EthAPIBackend) Miner() *miner.Miner { - return b.eth.Miner() -} - -func (b *EthAPIBackend) StartMining() error { - return b.eth.StartMining() -} - func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk) } diff --git a/eth/api_debug.go b/eth/api_debug.go index 05010a3969c6..d5e4dda1401c 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -56,7 +56,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { // If we're dumping the pending state, we need to request // both the pending block as well as the pending state from // the miner and operate on those - _, stateDb := api.eth.miner.Pending() + _, _, stateDb := api.eth.miner.Pending() if stateDb == nil { return state.Dump{}, errors.New("pending state is not available") } @@ -142,7 +142,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex // If we're dumping the pending state, we need to request // both the pending block as well as the pending state from // the miner and operate on those - _, stateDb = api.eth.miner.Pending() + _, _, stateDb = api.eth.miner.Pending() if stateDb == nil { return state.Dump{}, errors.New("pending state is not available") } diff --git a/eth/api_miner.go b/eth/api_miner.go index 764d0ae5e2f5..8c96f4c54aff 100644 --- a/eth/api_miner.go +++ b/eth/api_miner.go @@ -18,9 +18,7 @@ package eth import ( "math/big" - "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" ) @@ -34,21 +32,6 @@ func NewMinerAPI(e *Ethereum) *MinerAPI { return &MinerAPI{e} } -// Start starts the miner with the given number of threads. If threads is nil, -// the number of workers started is equal to the number of logical CPUs that are -// usable by this process. If mining is already running, this method adjust the -// number of threads allowed to use and updates the minimum price required by the -// transaction pool. -func (api *MinerAPI) Start() error { - return api.e.StartMining() -} - -// Stop terminates the miner, both at the consensus engine level as well as at -// the block creation level. -func (api *MinerAPI) Stop() { - api.e.StopMining() -} - // SetExtra sets the extra data string that is included when this miner mines a block. func (api *MinerAPI) SetExtra(extra string) (bool, error) { if err := api.e.Miner().SetExtra([]byte(extra)); err != nil { @@ -73,14 +56,3 @@ func (api *MinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool { api.e.Miner().SetGasCeil(uint64(gasLimit)) return true } - -// SetEtherbase sets the etherbase of the miner. -func (api *MinerAPI) SetEtherbase(etherbase common.Address) bool { - api.e.SetEtherbase(etherbase) - return true -} - -// SetRecommitInterval updates the interval for miner sealing work recommitting. -func (api *MinerAPI) SetRecommitInterval(interval int) { - api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond) -} diff --git a/eth/backend.go b/eth/backend.go index f6c1637acadf..81d84028a5f8 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -28,8 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/beacon" - "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" @@ -88,9 +86,8 @@ type Ethereum struct { APIBackend *EthAPIBackend - miner *miner.Miner - gasPrice *big.Int - etherbase common.Address + miner *miner.Miner + gasPrice *big.Int networkID uint64 netRPCService *ethapi.NetAPI @@ -164,7 +161,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { closeBloomHandler: make(chan struct{}), networkID: networkID, gasPrice: config.Miner.GasPrice, - etherbase: config.Miner.Etherbase, bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms), p2pServer: stack.Server(), @@ -211,7 +207,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if config.OverrideVerkle != nil { overrides.OverrideVerkle = config.OverrideVerkle } - eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TransactionHistory) + // TODO (MariusVanDerWijden) get rid of shouldPreserve in a follow-up PR + shouldPreserve := func(header *types.Header) bool { + return false + } + eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, shouldPreserve, &config.TransactionHistory) if err != nil { return nil, err } @@ -247,7 +247,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, &config.Miner, eth.blockchain.Config(), eth.EventMux(), eth.engine, eth.isLocalBlock) + eth.miner = miner.New(eth, config.Miner, eth.engine) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil} @@ -313,9 +313,6 @@ func (s *Ethereum) APIs() []rpc.API { // Append all the local APIs and return return append(apis, []rpc.API{ { - Namespace: "eth", - Service: NewEthereumAPI(s), - }, { Namespace: "miner", Service: NewMinerAPI(s), }, { @@ -338,138 +335,6 @@ func (s *Ethereum) ResetWithGenesisBlock(gb *types.Block) { s.blockchain.ResetWithGenesisBlock(gb) } -func (s *Ethereum) Etherbase() (eb common.Address, err error) { - s.lock.RLock() - etherbase := s.etherbase - s.lock.RUnlock() - - if etherbase != (common.Address{}) { - return etherbase, nil - } - return common.Address{}, errors.New("etherbase must be explicitly specified") -} - -// isLocalBlock checks whether the specified block is mined -// by local miner accounts. -// -// We regard two types of accounts as local miner account: etherbase -// and accounts specified via `txpool.locals` flag. -func (s *Ethereum) isLocalBlock(header *types.Header) bool { - author, err := s.engine.Author(header) - if err != nil { - log.Warn("Failed to retrieve block author", "number", header.Number.Uint64(), "hash", header.Hash(), "err", err) - return false - } - // Check whether the given address is etherbase. - s.lock.RLock() - etherbase := s.etherbase - s.lock.RUnlock() - if author == etherbase { - return true - } - // Check whether the given address is specified by `txpool.local` - // CLI flag. - for _, account := range s.config.TxPool.Locals { - if account == author { - return true - } - } - return false -} - -// shouldPreserve checks whether we should preserve the given block -// during the chain reorg depending on whether the author of block -// is a local account. -func (s *Ethereum) shouldPreserve(header *types.Header) bool { - // The reason we need to disable the self-reorg preserving for clique - // is it can be probable to introduce a deadlock. - // - // e.g. If there are 7 available signers - // - // r1 A - // r2 B - // r3 C - // r4 D - // r5 A [X] F G - // r6 [X] - // - // In the round5, the in-turn signer E is offline, so the worst case - // is A, F and G sign the block of round5 and reject the block of opponents - // and in the round6, the last available signer B is offline, the whole - // network is stuck. - if _, ok := s.engine.(*clique.Clique); ok { - return false - } - return s.isLocalBlock(header) -} - -// SetEtherbase sets the mining reward address. -func (s *Ethereum) SetEtherbase(etherbase common.Address) { - s.lock.Lock() - s.etherbase = etherbase - s.lock.Unlock() - - s.miner.SetEtherbase(etherbase) -} - -// StartMining starts the miner with the given number of CPU threads. If mining -// is already running, this method adjust the number of threads allowed to use -// and updates the minimum price required by the transaction pool. -func (s *Ethereum) StartMining() error { - // If the miner was not running, initialize it - if !s.IsMining() { - // Propagate the initial price point to the transaction pool - s.lock.RLock() - price := s.gasPrice - s.lock.RUnlock() - s.txPool.SetGasTip(price) - - // Configure the local mining address - eb, err := s.Etherbase() - if err != nil { - log.Error("Cannot start mining without etherbase", "err", err) - return fmt.Errorf("etherbase missing: %v", err) - } - var cli *clique.Clique - if c, ok := s.engine.(*clique.Clique); ok { - cli = c - } else if cl, ok := s.engine.(*beacon.Beacon); ok { - if c, ok := cl.InnerEngine().(*clique.Clique); ok { - cli = c - } - } - if cli != nil { - wallet, err := s.accountManager.Find(accounts.Account{Address: eb}) - if wallet == nil || err != nil { - log.Error("Etherbase account unavailable locally", "err", err) - return fmt.Errorf("signer missing: %v", err) - } - cli.Authorize(eb, wallet.SignData) - } - // If mining is started, we can disable the transaction rejection mechanism - // introduced to speed sync times. - s.handler.enableSyncedFeatures() - - go s.miner.Start() - } - return nil -} - -// StopMining terminates the miner, both at the consensus engine level as well as -// at the block creation level. -func (s *Ethereum) StopMining() { - // Update the thread count within the consensus engine - type threaded interface { - SetThreads(threads int) - } - if th, ok := s.engine.(threaded); ok { - th.SetThreads(-1) - } - // Stop the block creating itself - s.miner.Stop() -} - -func (s *Ethereum) IsMining() bool { return s.miner.Mining() } func (s *Ethereum) Miner() *miner.Miner { return s.miner } func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } @@ -531,7 +396,6 @@ func (s *Ethereum) Stop() error { s.bloomIndexer.Close() close(s.closeBloomHandler) s.txPool.Close() - s.miner.Close() s.blockchain.Stop() s.engine.Close() diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index a82e2d6cf6f2..a88996744c04 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -447,7 +447,9 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} + mcfg := miner.DefaultConfig + mcfg.PendingFeeRecipient = testAddr + ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: mcfg} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) @@ -460,7 +462,6 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't import test blocks:", err) } - ethservice.SetEtherbase(testAddr) ethservice.SetSynced() return n, ethservice } diff --git a/eth/catalyst/simulated_beacon_test.go b/eth/catalyst/simulated_beacon_test.go index 6fa97ad87a2a..df682b49d96e 100644 --- a/eth/catalyst/simulated_beacon_test.go +++ b/eth/catalyst/simulated_beacon_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" @@ -48,7 +49,7 @@ func startSimulatedBeaconEthService(t *testing.T, genesis *core.Genesis) (*node. t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} + ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: miner.DefaultConfig} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 83e3284a2b5b..f2b92d5a99da 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -333,7 +333,7 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*typ // pendingLogs returns the logs matching the filter criteria within the pending block. func (f *Filter) pendingLogs() []*types.Log { - block, receipts := f.sys.backend.PendingBlockAndReceipts() + block, receipts, _ := f.sys.backend.Pending() if block == nil || receipts == nil { return nil } diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index f98a1f84ce14..c32b837eb477 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -62,7 +63,7 @@ type Backend interface { GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) + Pending() (*types.Block, types.Receipts, *state.StateDB) CurrentHeader() *types.Header ChainConfig() *params.ChainConfig @@ -70,7 +71,6 @@ type Backend interface { SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription BloomStatus() (uint64, uint64) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) @@ -198,20 +198,18 @@ type EventSystem struct { lastHead *types.Header // Subscriptions - txsSub event.Subscription // Subscription for new transaction event - logsSub event.Subscription // Subscription for new log event - rmLogsSub event.Subscription // Subscription for removed log event - pendingLogsSub event.Subscription // Subscription for pending log event - chainSub event.Subscription // Subscription for new chain event + txsSub event.Subscription // Subscription for new transaction event + logsSub event.Subscription // Subscription for new log event + rmLogsSub event.Subscription // Subscription for removed log event + chainSub event.Subscription // Subscription for new chain event // Channels - install chan *subscription // install filter for event notification - uninstall chan *subscription // remove filter for event notification - txsCh chan core.NewTxsEvent // Channel to receive new transactions event - logsCh chan []*types.Log // Channel to receive new log event - pendingLogsCh chan []*types.Log // Channel to receive new log event - rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event - chainCh chan core.ChainEvent // Channel to receive new chain event + install chan *subscription // install filter for event notification + uninstall chan *subscription // remove filter for event notification + txsCh chan core.NewTxsEvent // Channel to receive new transactions event + logsCh chan []*types.Log // Channel to receive new log event + rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event + chainCh chan core.ChainEvent // Channel to receive new chain event } // NewEventSystem creates a new manager that listens for event on the given mux, @@ -222,16 +220,15 @@ type EventSystem struct { // or by stopping the given mux. func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m := &EventSystem{ - sys: sys, - backend: sys.backend, - lightMode: lightMode, - install: make(chan *subscription), - uninstall: make(chan *subscription), - txsCh: make(chan core.NewTxsEvent, txChanSize), - logsCh: make(chan []*types.Log, logsChanSize), - rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), - pendingLogsCh: make(chan []*types.Log, logsChanSize), - chainCh: make(chan core.ChainEvent, chainEvChanSize), + sys: sys, + backend: sys.backend, + lightMode: lightMode, + install: make(chan *subscription), + uninstall: make(chan *subscription), + txsCh: make(chan core.NewTxsEvent, txChanSize), + logsCh: make(chan []*types.Log, logsChanSize), + rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), + chainCh: make(chan core.ChainEvent, chainEvChanSize), } // Subscribe events @@ -239,10 +236,9 @@ func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m.logsSub = m.backend.SubscribeLogsEvent(m.logsCh) m.rmLogsSub = m.backend.SubscribeRemovedLogsEvent(m.rmLogsCh) m.chainSub = m.backend.SubscribeChainEvent(m.chainCh) - m.pendingLogsSub = m.backend.SubscribePendingLogsEvent(m.pendingLogsCh) // Make sure none of the subscriptions are empty - if m.txsSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil || m.pendingLogsSub == nil { + if m.txsSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil { log.Crit("Subscribe for event system failed") } @@ -434,12 +430,12 @@ func (es *EventSystem) handleLogs(filters filterIndex, ev []*types.Log) { } } -func (es *EventSystem) handlePendingLogs(filters filterIndex, ev []*types.Log) { - if len(ev) == 0 { +func (es *EventSystem) handlePendingLogs(filters filterIndex, logs []*types.Log) { + if len(logs) == 0 { return } for _, f := range filters[PendingLogsSubscription] { - matchedLogs := filterLogs(ev, nil, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics) + matchedLogs := filterLogs(logs, nil, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics) if len(matchedLogs) > 0 { f.logs <- matchedLogs } @@ -550,7 +546,6 @@ func (es *EventSystem) eventLoop() { es.txsSub.Unsubscribe() es.logsSub.Unsubscribe() es.rmLogsSub.Unsubscribe() - es.pendingLogsSub.Unsubscribe() es.chainSub.Unsubscribe() }() @@ -567,10 +562,29 @@ func (es *EventSystem) eventLoop() { es.handleLogs(index, ev) case ev := <-es.rmLogsCh: es.handleLogs(index, ev.Logs) - case ev := <-es.pendingLogsCh: - es.handlePendingLogs(index, ev) case ev := <-es.chainCh: es.handleChainEvent(index, ev) + // If we have no pending log subscription, + // we don't need to collect any pending logs. + if len(index[PendingLogsSubscription]) == 0 { + continue + } + + // Pull the pending logs if there is a new chain head. + pendingBlock, pendingReceipts, _ := es.backend.Pending() + if pendingBlock == nil || pendingReceipts == nil { + continue + } + if pendingBlock.ParentHash() != ev.Block.Hash() { + continue + } + var logs []*types.Log + for _, receipt := range pendingReceipts { + if len(receipt.Logs) > 0 { + logs = append(logs, receipt.Logs...) + } + } + es.handlePendingLogs(index, logs) case f := <-es.install: if f.typ == MinedAndPendingLogsSubscription { diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 99c012cc84f4..6238c9773522 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -48,7 +49,6 @@ type testBackend struct { txFeed event.Feed logsFeed event.Feed rmLogsFeed event.Feed - pendingLogsFeed event.Feed chainFeed event.Feed pendingBlock *types.Block pendingReceipts types.Receipts @@ -125,8 +125,8 @@ func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash, number uint return logs, nil } -func (b *testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { - return b.pendingBlock, b.pendingReceipts +func (b *testBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { + return b.pendingBlock, b.pendingReceipts, nil } func (b *testBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { @@ -141,10 +141,6 @@ func (b *testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript return b.logsFeed.Subscribe(ch) } -func (b *testBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - return b.pendingLogsFeed.Subscribe(ch) -} - func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.chainFeed.Subscribe(ch) } @@ -180,6 +176,20 @@ func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.Matc }() } +func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) { + b.pendingBlock = block + b.pendingReceipts = receipts +} + +func (b *testBackend) notifyPending(logs []*types.Log) { + genesis := &core.Genesis{ + Config: params.TestChainConfig, + } + _, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 2, func(i int, b *core.BlockGen) {}) + b.setPending(blocks[1], []*types.Receipt{{Logs: logs}}) + b.chainFeed.Send(core.ChainEvent{Block: blocks[0]}) +} + func newTestFilterSystem(t testing.TB, db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { backend := &testBackend{db: db} sys := NewFilterSystem(backend, cfg) @@ -203,7 +213,7 @@ func TestBlockSubscription(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), } _, chain, _ = core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 10, func(i int, gen *core.BlockGen) {}) - chainEvents = []core.ChainEvent{} + chainEvents []core.ChainEvent ) for _, blk := range chain { @@ -386,7 +396,7 @@ func TestLogFilterCreation(t *testing.T) { {FilterCriteria{FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(100)}, false}, // from block "higher" than to block {FilterCriteria{FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, false}, - // topics more then 4 + // topics more than 4 {FilterCriteria{Topics: [][]common.Hash{{}, {}, {}, {}, {}}}, false}, } ) @@ -546,9 +556,9 @@ func TestLogFilter(t *testing.T) { if nsend := backend.logsFeed.Send(allLogs); nsend == 0 { t.Fatal("Logs event not delivered") } - if nsend := backend.pendingLogsFeed.Send(allLogs); nsend == 0 { - t.Fatal("Pending logs event not delivered") - } + + // set pending logs + backend.notifyPending(allLogs) for i, tt := range testCases { var fetched []*types.Log @@ -754,10 +764,12 @@ func TestPendingLogsSubscription(t *testing.T) { }() } - // raise events - for _, ev := range allLogs { - backend.pendingLogsFeed.Send(ev) + // set pending logs + var flattenLogs []*types.Log + for _, logs := range allLogs { + flattenLogs = append(flattenLogs, logs...) } + backend.notifyPending(flattenLogs) for i := range testCases { err := <-testCases[i].err diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 659ca5ce197d..48aaa584dbb1 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -109,8 +109,8 @@ func BenchmarkFilters(b *testing.B) { func TestFilters(t *testing.T) { var ( - db = rawdb.NewMemoryDatabase() - _, sys = newTestFilterSystem(t, db, Config{}) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) // Sender account key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) @@ -277,8 +277,7 @@ func TestFilters(t *testing.T) { }), signer, key1) gen.AddTx(tx) }) - sys.backend.(*testBackend).pendingBlock = pchain[0] - sys.backend.(*testBackend).pendingReceipts = preceipts[0] + backend.setPending(pchain[0], preceipts[0]) for i, tc := range []struct { f *Filter diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index d657eb6d996b..8ab57294b7e8 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -160,7 +160,7 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, reqEnd rpc.BlockNum ) switch reqEnd { case rpc.PendingBlockNumber: - if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { + if pendingBlock, pendingReceipts, _ = oracle.backend.Pending(); pendingBlock != nil { resolved = pendingBlock.Header() } else { // Pending block not supported by backend, process only until latest block. diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index b71964981145..3fa70e41a094 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" @@ -54,7 +55,7 @@ type OracleBackend interface { HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) + Pending() (*types.Block, types.Receipts, *state.StateDB) ChainConfig() *params.ChainConfig SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription } diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 79217502f799..1d2e02cde6e1 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -97,12 +98,13 @@ func (b *testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types. return b.chain.GetReceiptsByHash(hash), nil } -func (b *testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { +func (b *testBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { if b.pending { block := b.chain.GetBlockByNumber(testHead + 1) - return block, b.chain.GetReceiptsByHash(block.Hash()) + state, _ := b.chain.StateAt(block.Root()) + return block, b.chain.GetReceiptsByHash(block.Hash()), state } - return nil, nil + return nil, nil, nil } func (b *testBackend) ChainConfig() *params.ChainConfig { diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 0d2675f8d10d..2f3229cedcb5 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -602,17 +602,22 @@ func testAtFunctions(t *testing.T, client *rpc.Client) { } // send a transaction for some interesting pending status + // and wait for the transaction to be included in the pending block sendTransaction(ec) - time.Sleep(100 * time.Millisecond) - // Check pending transaction count - pending, err := ec.PendingTransactionCount(context.Background()) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if pending != 1 { - t.Fatalf("unexpected pending, wanted 1 got: %v", pending) + // wait for the transaction to be included in the pending block + for { + // Check pending transaction count + pending, err := ec.PendingTransactionCount(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if pending == 1 { + break + } + time.Sleep(100 * time.Millisecond) } + // Query balance balance, err := ec.BalanceAt(context.Background(), testAddr, nil) if err != nil { @@ -737,7 +742,7 @@ func sendTransaction(ec *Client) error { if err != nil { return err } - nonce, err := ec.PendingNonceAt(context.Background(), testAddr) + nonce, err := ec.NonceAt(context.Background(), testAddr, nil) if err != nil { return err } diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 61ceec443ebc..6e71666ec121 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -39,7 +39,6 @@ import ( ethproto "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" @@ -80,13 +79,6 @@ type fullNodeBackend interface { SuggestGasTipCap(ctx context.Context) (*big.Int, error) } -// miningNodeBackend encompasses the functionality necessary for a mining node -// reporting to ethstats -type miningNodeBackend interface { - fullNodeBackend - Miner() *miner.Miner -} - // Service implements an Ethereum netstats reporting daemon that pushes local // chain statistics up to a monitoring server. type Service struct { @@ -777,30 +769,21 @@ func (s *Service) reportPending(conn *connWrapper) error { type nodeStats struct { Active bool `json:"active"` Syncing bool `json:"syncing"` - Mining bool `json:"mining"` - Hashrate int `json:"hashrate"` Peers int `json:"peers"` GasPrice int `json:"gasPrice"` Uptime int `json:"uptime"` } -// reportStats retrieves various stats about the node at the networking and -// mining layer and reports it to the stats server. +// reportStats retrieves various stats about the node at the networking layer +// and reports it to the stats server. func (s *Service) reportStats(conn *connWrapper) error { - // Gather the syncing and mining infos from the local miner instance + // Gather the syncing infos from the local miner instance var ( - mining bool - hashrate int syncing bool gasprice int ) // check if backend is a full node if fullBackend, ok := s.backend.(fullNodeBackend); ok { - if miningBackend, ok := s.backend.(miningNodeBackend); ok { - mining = miningBackend.Miner().Mining() - hashrate = int(miningBackend.Miner().Hashrate()) - } - sync := fullBackend.SyncProgress() syncing = !sync.Done() @@ -820,8 +803,6 @@ func (s *Service) reportStats(conn *connWrapper) error { "id": s.node, "stats": &nodeStats{ Active: true, - Mining: mining, - Hashrate: hashrate, Peers: s.server.PeerCount(), GasPrice: gasprice, Syncing: syncing, diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 1d0383daad03..3f69f861444c 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -547,7 +547,7 @@ func (b testBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOr } panic("only implemented for number") } -func (b testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { panic("implement me") } +func (b testBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { panic("implement me") } func (b testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { header, err := b.HeaderByHash(ctx, hash) if header == nil || err != nil { @@ -615,9 +615,6 @@ func (b testBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { panic("implement me") } -func (b testBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - panic("implement me") -} func (b testBackend) BloomStatus() (uint64, uint64) { panic("implement me") } func (b testBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { panic("implement me") diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 5f408ba20ba5..fd2f5699eabf 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -65,7 +65,7 @@ type Backend interface { BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) + Pending() (*types.Block, types.Receipts, *state.StateDB) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) GetTd(ctx context.Context, hash common.Hash) *big.Int GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM @@ -94,7 +94,6 @@ type Backend interface { GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription BloomStatus() (uint64, uint64) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) } diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index 1b1634b25031..24ecb1dee4e7 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -358,7 +358,7 @@ func (b *backendMock) StateAndHeaderByNumber(ctx context.Context, number rpc.Blo func (b *backendMock) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { return nil, nil, nil } -func (b *backendMock) PendingBlockAndReceipts() (*types.Block, types.Receipts) { return nil, nil } +func (b *backendMock) Pending() (*types.Block, types.Receipts, *state.StateDB) { return nil, nil, nil } func (b *backendMock) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { return nil, nil } @@ -396,9 +396,6 @@ func (b *backendMock) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscr func (b *backendMock) BloomStatus() (uint64, uint64) { return 0, 0 } func (b *backendMock) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {} func (b *backendMock) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return nil } -func (b *backendMock) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - return nil -} func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return nil } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index b86b5909d2cb..1da7d737dd94 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -649,20 +649,6 @@ const MinerJs = ` web3._extend({ property: 'miner', methods: [ - new web3._extend.Method({ - name: 'start', - call: 'miner_start', - }), - new web3._extend.Method({ - name: 'stop', - call: 'miner_stop' - }), - new web3._extend.Method({ - name: 'setEtherbase', - call: 'miner_setEtherbase', - params: 1, - inputFormatter: [web3._extend.formatters.inputAddressFormatter] - }), new web3._extend.Method({ name: 'setExtra', call: 'miner_setExtra', @@ -680,15 +666,6 @@ web3._extend({ params: 1, inputFormatter: [web3._extend.utils.fromDecimal] }), - new web3._extend.Method({ - name: 'setRecommitInterval', - call: 'miner_setRecommitInterval', - params: 1, - }), - new web3._extend.Method({ - name: 'getHashrate', - call: 'miner_getHashrate' - }), ], properties: [] }); diff --git a/miner/miner.go b/miner/miner.go index 58bb71b557b8..430efcb2fcf1 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -30,9 +30,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) @@ -45,207 +42,124 @@ type Backend interface { // Config is the configuration parameters of mining. type Config struct { - Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards - ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner - GasFloor uint64 // Target gas floor for mined blocks. - GasCeil uint64 // Target gas ceiling for mined blocks. - GasPrice *big.Int // Minimum gas price for mining a transaction - Recommit time.Duration // The time interval for miner to re-create mining work. - - NewPayloadTimeout time.Duration // The maximum time allowance for creating a new payload + Etherbase common.Address `toml:"-"` // Deprecated + PendingFeeRecipient common.Address `toml:"-"` // Address for pending block rewards. + ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner + GasCeil uint64 // Target gas ceiling for mined blocks. + GasPrice *big.Int // Minimum gas price for mining a transaction + Recommit time.Duration // The time interval for miner to re-create mining work. } // DefaultConfig contains default settings for miner. var DefaultConfig = Config{ - GasCeil: 30000000, + GasCeil: 30_000_000, GasPrice: big.NewInt(params.GWei), // The default recommit time is chosen as two seconds since // consensus-layer usually will wait a half slot of time(6s) // for payload generation. It should be enough for Geth to // run 3 rounds. - Recommit: 2 * time.Second, - NewPayloadTimeout: 2 * time.Second, + Recommit: 2 * time.Second, } -// Miner creates blocks and searches for proof-of-work values. +// Miner is the main object which takes care of submitting new work to consensus +// engine and gathering the sealing result. type Miner struct { - mux *event.TypeMux - eth Backend - engine consensus.Engine - exitCh chan struct{} - startCh chan struct{} - stopCh chan struct{} - worker *worker - - wg sync.WaitGroup -} - -func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(header *types.Header) bool) *Miner { - miner := &Miner{ - mux: mux, - eth: eth, - engine: engine, - exitCh: make(chan struct{}), - startCh: make(chan struct{}), - stopCh: make(chan struct{}), - worker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true), - } - miner.wg.Add(1) - go miner.update() - return miner -} - -// update keeps track of the downloader events. Please be aware that this is a one shot type of update loop. -// It's entered once and as soon as `Done` or `Failed` has been broadcasted the events are unregistered and -// the loop is exited. This to prevent a major security vuln where external parties can DOS you with blocks -// and halt your mining operation for as long as the DOS continues. -func (miner *Miner) update() { - defer miner.wg.Done() - - events := miner.mux.Subscribe(downloader.StartEvent{}, downloader.DoneEvent{}, downloader.FailedEvent{}) - defer func() { - if !events.Closed() { - events.Unsubscribe() - } - }() - - shouldStart := false - canStart := true - dlEventCh := events.Chan() - for { - select { - case ev := <-dlEventCh: - if ev == nil { - // Unsubscription done, stop listening - dlEventCh = nil - continue - } - switch ev.Data.(type) { - case downloader.StartEvent: - wasMining := miner.Mining() - miner.worker.stop() - canStart = false - if wasMining { - // Resume mining after sync was finished - shouldStart = true - log.Info("Mining aborted due to sync") - } - miner.worker.syncing.Store(true) - - case downloader.FailedEvent: - canStart = true - if shouldStart { - miner.worker.start() - } - miner.worker.syncing.Store(false) - - case downloader.DoneEvent: - canStart = true - if shouldStart { - miner.worker.start() - } - miner.worker.syncing.Store(false) - - // Stop reacting to downloader events - events.Unsubscribe() - } - case <-miner.startCh: - if canStart { - miner.worker.start() - } - shouldStart = true - case <-miner.stopCh: - shouldStart = false - miner.worker.stop() - case <-miner.exitCh: - miner.worker.close() - return - } + confMu sync.RWMutex // The lock used to protect the config fields: GasCeil, GasTip and Extradata + config *Config + chainConfig *params.ChainConfig + engine consensus.Engine + txpool *txpool.TxPool + chain *core.BlockChain + pending *pending + pendingMu sync.Mutex // Lock protects the pending block +} + +// New creates a new miner with provided config. +func New(eth Backend, config Config, engine consensus.Engine) *Miner { + return &Miner{ + config: &config, + chainConfig: eth.BlockChain().Config(), + engine: engine, + txpool: eth.TxPool(), + chain: eth.BlockChain(), + pending: &pending{}, } } -func (miner *Miner) Start() { - miner.startCh <- struct{}{} -} - -func (miner *Miner) Stop() { - miner.stopCh <- struct{}{} -} - -func (miner *Miner) Close() { - close(miner.exitCh) - miner.wg.Wait() -} - -func (miner *Miner) Mining() bool { - return miner.worker.isRunning() -} - -func (miner *Miner) Hashrate() uint64 { - if pow, ok := miner.engine.(consensus.PoW); ok { - return uint64(pow.Hashrate()) +// Pending returns the currently pending block and associated receipts, logs +// and statedb. The returned values can be nil in case the pending block is +// not initialized. +func (miner *Miner) Pending() (*types.Block, types.Receipts, *state.StateDB) { + pending := miner.getPending() + if pending == nil { + return nil, nil, nil } - return 0 + return pending.block, pending.receipts, pending.stateDB.Copy() } +// SetExtra sets the content used to initialize the block extra field. func (miner *Miner) SetExtra(extra []byte) error { if uint64(len(extra)) > params.MaximumExtraDataSize { return fmt.Errorf("extra exceeds max length. %d > %v", len(extra), params.MaximumExtraDataSize) } - miner.worker.setExtra(extra) + miner.confMu.Lock() + miner.config.ExtraData = extra + miner.confMu.Unlock() return nil } -func (miner *Miner) SetGasTip(tip *big.Int) error { - miner.worker.setGasTip(tip) - return nil -} - -// SetRecommitInterval sets the interval for sealing work resubmitting. -func (miner *Miner) SetRecommitInterval(interval time.Duration) { - miner.worker.setRecommitInterval(interval) -} - -// Pending returns the currently pending block and associated state. The returned -// values can be nil in case the pending block is not initialized -func (miner *Miner) Pending() (*types.Block, *state.StateDB) { - return miner.worker.pending() -} - -// PendingBlock returns the currently pending block. The returned block can be -// nil in case the pending block is not initialized. -// -// Note, to access both the pending block and the pending state -// simultaneously, please use Pending(), as the pending state can -// change between multiple method calls -func (miner *Miner) PendingBlock() *types.Block { - return miner.worker.pendingBlock() -} - -// PendingBlockAndReceipts returns the currently pending block and corresponding receipts. -// The returned values can be nil in case the pending block is not initialized. -func (miner *Miner) PendingBlockAndReceipts() (*types.Block, types.Receipts) { - return miner.worker.pendingBlockAndReceipts() -} - -func (miner *Miner) SetEtherbase(addr common.Address) { - miner.worker.setEtherbase(addr) -} - // SetGasCeil sets the gaslimit to strive for when mining blocks post 1559. // For pre-1559 blocks, it sets the ceiling. func (miner *Miner) SetGasCeil(ceil uint64) { - miner.worker.setGasCeil(ceil) + miner.confMu.Lock() + miner.config.GasCeil = ceil + miner.confMu.Unlock() } -// SubscribePendingLogs starts delivering logs from pending transactions -// to the given channel. -func (miner *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription { - return miner.worker.pendingLogsFeed.Subscribe(ch) +// SetGasTip sets the minimum gas tip for inclusion. +func (miner *Miner) SetGasTip(tip *big.Int) error { + miner.confMu.Lock() + miner.config.GasPrice = tip + miner.confMu.Unlock() + return nil } // BuildPayload builds the payload according to the provided parameters. func (miner *Miner) BuildPayload(args *BuildPayloadArgs) (*Payload, error) { - return miner.worker.buildPayload(args) + return miner.buildPayload(args) +} + +// getPending retrieves the pending block based on the current head block. +// The result might be nil if pending generation is failed. +func (miner *Miner) getPending() *newPayloadResult { + header := miner.chain.CurrentHeader() + miner.pendingMu.Lock() + defer miner.pendingMu.Unlock() + if cached := miner.pending.resolve(header.Hash()); cached != nil { + return cached + } + + var ( + timestamp = uint64(time.Now().Unix()) + withdrawal types.Withdrawals + ) + if miner.chainConfig.IsShanghai(new(big.Int).Add(header.Number, big.NewInt(1)), timestamp) { + withdrawal = []*types.Withdrawal{} + } + ret := miner.generateWork(&generateParams{ + timestamp: timestamp, + forceTime: false, + parentHash: header.Hash(), + coinbase: miner.config.PendingFeeRecipient, + random: common.Hash{}, + withdrawals: withdrawal, + beaconRoot: nil, + noTxs: false, + }) + if ret.err != nil { + return nil + } + miner.pending.update(header.Hash(), ret) + return ret } diff --git a/miner/miner_test.go b/miner/miner_test.go index 5907fb446466..7c39564240c1 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -18,10 +18,9 @@ package miner import ( - "errors" "math/big" + "sync" "testing" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/clique" @@ -33,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" @@ -60,10 +58,6 @@ func (m *mockBackend) TxPool() *txpool.TxPool { return m.txPool } -func (m *mockBackend) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) { - return nil, errors.New("not supported") -} - type testBlockChain struct { root common.Hash config *params.ChainConfig @@ -99,171 +93,18 @@ func (bc *testBlockChain) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) return bc.chainHeadFeed.Subscribe(ch) } -func TestMiner(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - // Subsequent downloader events after a successful DoneEvent should not cause the - // miner to start or stop. This prevents a security vulnerability - // that would allow entities to present fake high blocks that would - // stop mining operations by causing a downloader sync - // until it was discovered they were invalid, whereon mining would resume. - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, true) - - mux.Post(downloader.FailedEvent{}) - waitForMiningState(t, miner, true) -} - -// TestMinerDownloaderFirstFails tests that mining is only -// permitted to run indefinitely once the downloader sees a DoneEvent (success). -// An initial FailedEvent should allow mining to stop on a subsequent -// downloader StartEvent. -func TestMinerDownloaderFirstFails(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.FailedEvent{}) - waitForMiningState(t, miner, true) - - // Since the downloader hasn't yet emitted a successful DoneEvent, - // we expect the miner to stop on next StartEvent. - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - - // Downloader finally succeeds. - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - // Downloader starts again. - // Since it has achieved a DoneEvent once, we expect miner - // state to be unchanged. - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, true) - - mux.Post(downloader.FailedEvent{}) - waitForMiningState(t, miner, true) -} - -func TestMinerStartStopAfterDownloaderEvents(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - - // Downloader finally succeeds. - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - miner.Stop() - waitForMiningState(t, miner, false) - - miner.Start() - waitForMiningState(t, miner, true) - - miner.Stop() - waitForMiningState(t, miner, false) -} - -func TestStartWhileDownload(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - waitForMiningState(t, miner, false) - miner.Start() - waitForMiningState(t, miner, true) - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - // Starting the miner after the downloader should not work - miner.Start() - waitForMiningState(t, miner, false) -} - -func TestStartStopMiner(t *testing.T) { - t.Parallel() - miner, _, cleanup := createMiner(t) - defer cleanup(false) - waitForMiningState(t, miner, false) - miner.Start() - waitForMiningState(t, miner, true) - miner.Stop() - waitForMiningState(t, miner, false) -} - -func TestCloseMiner(t *testing.T) { - t.Parallel() - miner, _, cleanup := createMiner(t) - defer cleanup(true) - waitForMiningState(t, miner, false) - miner.Start() - waitForMiningState(t, miner, true) - // Terminate the miner and wait for the update loop to run - miner.Close() - waitForMiningState(t, miner, false) -} - -// TestMinerSetEtherbase checks that etherbase becomes set even if mining isn't -// possible at the moment -func TestMinerSetEtherbase(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - // Now user tries to configure proper mining address - miner.Start() - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - coinbase := common.HexToAddress("0xdeedbeef") - miner.SetEtherbase(coinbase) - if addr := miner.worker.etherbase(); addr != coinbase { - t.Fatalf("Unexpected etherbase want %x got %x", coinbase, addr) - } -} - -// waitForMiningState waits until either -// * the desired mining state was reached -// * a timeout was reached which fails the test -func waitForMiningState(t *testing.T, m *Miner, mining bool) { - t.Helper() - - var state bool - for i := 0; i < 100; i++ { - time.Sleep(10 * time.Millisecond) - if state = m.Mining(); state == mining { - return +func TestBuildPendingBlocks(t *testing.T) { + miner := createMiner(t) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + block, _, _ := miner.Pending() + if block == nil { + t.Error("Pending failed") } - } - t.Fatalf("Mining() == %t, want %t", state, mining) + }() + wg.Wait() } func minerTestGenesisBlock(period uint64, gasLimit uint64, faucet common.Address) *core.Genesis { @@ -294,10 +135,11 @@ func minerTestGenesisBlock(period uint64, gasLimit uint64, faucet common.Address }, } } -func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { + +func createMiner(t *testing.T) *Miner { // Create Ethash config config := Config{ - Etherbase: common.HexToAddress("123456789"), + PendingFeeRecipient: common.HexToAddress("123456789"), } // Create chainConfig chainDB := rawdb.NewMemoryDatabase() @@ -320,18 +162,8 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { pool := legacypool.New(testTxPoolConfig, blockchain) txpool, _ := txpool.New(testTxPoolConfig.PriceLimit, blockchain, []txpool.SubPool{pool}) - backend := NewMockBackend(bc, txpool) - // Create event Mux - mux := new(event.TypeMux) // Create Miner - miner := New(backend, &config, chainConfig, mux, engine, nil) - cleanup := func(skipMiner bool) { - bc.Stop() - engine.Close() - txpool.Close() - if !skipMiner { - miner.Close() - } - } - return miner, mux, cleanup + backend := NewMockBackend(bc, txpool) + miner := New(backend, config, engine) + return miner } diff --git a/miner/payload_building.go b/miner/payload_building.go index 719736c4795c..cbdb82a642cf 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -46,7 +46,6 @@ type BuildPayloadArgs struct { // Id computes an 8-byte identifier by hashing the components of the payload arguments. func (args *BuildPayloadArgs) Id() engine.PayloadID { - // Hash hasher := sha256.New() hasher.Write(args.Parent[:]) binary.Write(hasher, binary.BigEndian, args.Timestamp) @@ -177,7 +176,7 @@ func (payload *Payload) ResolveFull() *engine.ExecutionPayloadEnvelope { } // buildPayload builds the payload according to the provided parameters. -func (w *worker) buildPayload(args *BuildPayloadArgs) (*Payload, error) { +func (miner *Miner) buildPayload(args *BuildPayloadArgs) (*Payload, error) { // Build the initial version with no transaction included. It should be fast // enough to run. The empty payload can at least make sure there is something // to deliver for not missing slot. @@ -191,7 +190,7 @@ func (w *worker) buildPayload(args *BuildPayloadArgs) (*Payload, error) { beaconRoot: args.BeaconRoot, noTxs: true, } - empty := w.getSealingBlock(emptyParams) + empty := miner.generateWork(emptyParams) if empty.err != nil { return nil, empty.err } @@ -227,11 +226,11 @@ func (w *worker) buildPayload(args *BuildPayloadArgs) (*Payload, error) { select { case <-timer.C: start := time.Now() - r := w.getSealingBlock(fullParams) + r := miner.generateWork(fullParams) if r.err == nil { payload.update(r, time.Since(start)) } - timer.Reset(w.recommit) + timer.Reset(miner.config.Recommit) case <-payload.stop: log.Info("Stopping work on payload", "id", payload.id, "reason", "delivery") return diff --git a/miner/payload_building_test.go b/miner/payload_building_test.go index 708072b5ecf2..1728b9e5bd59 100644 --- a/miner/payload_building_test.go +++ b/miner/payload_building_test.go @@ -17,26 +17,141 @@ package miner import ( + "math/big" "reflect" "testing" "time" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) +var ( + // Test chain configurations + testTxPoolConfig legacypool.Config + ethashChainConfig *params.ChainConfig + cliqueChainConfig *params.ChainConfig + + // Test accounts + testBankKey, _ = crypto.GenerateKey() + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) + testBankFunds = big.NewInt(1000000000000000000) + + testUserKey, _ = crypto.GenerateKey() + testUserAddress = crypto.PubkeyToAddress(testUserKey.PublicKey) + + // Test transactions + pendingTxs []*types.Transaction + newTxs []*types.Transaction + + testConfig = Config{ + PendingFeeRecipient: testBankAddress, + Recommit: time.Second, + GasCeil: params.GenesisGasLimit, + } +) + +func init() { + testTxPoolConfig = legacypool.DefaultConfig + testTxPoolConfig.Journal = "" + ethashChainConfig = new(params.ChainConfig) + *ethashChainConfig = *params.TestChainConfig + cliqueChainConfig = new(params.ChainConfig) + *cliqueChainConfig = *params.TestChainConfig + cliqueChainConfig.Clique = ¶ms.CliqueConfig{ + Period: 10, + Epoch: 30000, + } + + signer := types.LatestSigner(params.TestChainConfig) + tx1 := types.MustSignNewTx(testBankKey, signer, &types.AccessListTx{ + ChainID: params.TestChainConfig.ChainID, + Nonce: 0, + To: &testUserAddress, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + pendingTxs = append(pendingTxs, tx1) + + tx2 := types.MustSignNewTx(testBankKey, signer, &types.LegacyTx{ + Nonce: 1, + To: &testUserAddress, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + newTxs = append(newTxs, tx2) +} + +// testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing. +type testWorkerBackend struct { + db ethdb.Database + txPool *txpool.TxPool + chain *core.BlockChain + genesis *core.Genesis +} + +func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, n int) *testWorkerBackend { + var gspec = &core.Genesis{ + Config: chainConfig, + Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + } + switch e := engine.(type) { + case *clique.Clique: + gspec.ExtraData = make([]byte, 32+common.AddressLength+crypto.SignatureLength) + copy(gspec.ExtraData[32:32+common.AddressLength], testBankAddress.Bytes()) + e.Authorize(testBankAddress, func(account accounts.Account, s string, data []byte) ([]byte, error) { + return crypto.Sign(crypto.Keccak256(data), testBankKey) + }) + case *ethash.Ethash: + default: + t.Fatalf("unexpected consensus engine type: %T", engine) + } + chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec, nil, engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("core.NewBlockChain failed: %v", err) + } + pool := legacypool.New(testTxPoolConfig, chain) + txpool, _ := txpool.New(testTxPoolConfig.PriceLimit, chain, []txpool.SubPool{pool}) + + return &testWorkerBackend{ + db: db, + chain: chain, + txPool: txpool, + genesis: gspec, + } +} + +func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } +func (b *testWorkerBackend) TxPool() *txpool.TxPool { return b.txPool } + +func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*Miner, *testWorkerBackend) { + backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) + backend.txPool.Add(pendingTxs, true, false) + w := New(backend, testConfig, engine) + return w, backend +} + func TestBuildPayload(t *testing.T) { - t.Parallel() var ( db = rawdb.NewMemoryDatabase() recipient = common.HexToAddress("0xdeadbeef") ) w, b := newTestWorker(t, params.TestChainConfig, ethash.NewFaker(), db, 0) - defer w.close() timestamp := uint64(time.Now().Unix()) args := &BuildPayloadArgs{ diff --git a/miner/pending.go b/miner/pending.go new file mode 100644 index 000000000000..bb91fe89690a --- /dev/null +++ b/miner/pending.go @@ -0,0 +1,67 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package miner + +import ( + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" +) + +// pendingTTL indicates the period of time a generated pending block should +// exist to serve RPC requests before being discarded if the parent block +// has not changed yet. The value is chosen to align with the recommit interval. +const pendingTTL = 2 * time.Second + +// pending wraps a pending block with additional metadata. +type pending struct { + created time.Time + parentHash common.Hash + result *newPayloadResult + lock sync.Mutex +} + +// resolve retrieves the cached pending result if it's available. Nothing will be +// returned if the parentHash is not matched or the result is already too old. +// +// Note, don't modify the returned payload result. +func (p *pending) resolve(parentHash common.Hash) *newPayloadResult { + p.lock.Lock() + defer p.lock.Unlock() + + if p.result == nil { + return nil + } + if parentHash != p.parentHash { + return nil + } + if time.Since(p.created) > pendingTTL { + return nil + } + return p.result +} + +// update refreshes the cached pending block with newly created one. +func (p *pending) update(parent common.Hash, result *newPayloadResult) { + p.lock.Lock() + defer p.lock.Unlock() + + p.parentHash = parent + p.result = result + p.created = time.Now() +} diff --git a/miner/stress/clique/main.go b/miner/stress/clique/main.go deleted file mode 100644 index 60593938458b..000000000000 --- a/miner/stress/clique/main.go +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// This file contains a miner stress test based on the Clique consensus engine. -package main - -import ( - "bytes" - "crypto/ecdsa" - "math/big" - "math/rand" - "os" - "os/signal" - "time" - - "github.com/ethereum/go-ethereum/accounts/keystore" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/fdlimit" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/txpool/legacypool" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/miner" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/params" -) - -func main() { - log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) - fdlimit.Raise(2048) - - // Generate a batch of accounts to seal and fund with - faucets := make([]*ecdsa.PrivateKey, 128) - for i := 0; i < len(faucets); i++ { - faucets[i], _ = crypto.GenerateKey() - } - sealers := make([]*ecdsa.PrivateKey, 4) - for i := 0; i < len(sealers); i++ { - sealers[i], _ = crypto.GenerateKey() - } - // Create a Clique network based off of the Sepolia config - genesis := makeGenesis(faucets, sealers) - - // Handle interrupts. - interruptCh := make(chan os.Signal, 5) - signal.Notify(interruptCh, os.Interrupt) - - var ( - stacks []*node.Node - nodes []*eth.Ethereum - enodes []*enode.Node - ) - for _, sealer := range sealers { - // Start the node and wait until it's up - stack, ethBackend, err := makeSealer(genesis) - if err != nil { - panic(err) - } - defer stack.Close() - - for stack.Server().NodeInfo().Ports.Listener == 0 { - time.Sleep(250 * time.Millisecond) - } - // Connect the node to all the previous ones - for _, n := range enodes { - stack.Server().AddPeer(n) - } - // Start tracking the node and its enode - stacks = append(stacks, stack) - nodes = append(nodes, ethBackend) - enodes = append(enodes, stack.Server().Self()) - - // Inject the signer key and start sealing with it - ks := keystore.NewKeyStore(stack.KeyStoreDir(), keystore.LightScryptN, keystore.LightScryptP) - signer, err := ks.ImportECDSA(sealer, "") - if err != nil { - panic(err) - } - if err := ks.Unlock(signer, ""); err != nil { - panic(err) - } - stack.AccountManager().AddBackend(ks) - } - - // Iterate over all the nodes and start signing on them - time.Sleep(3 * time.Second) - for _, node := range nodes { - if err := node.StartMining(); err != nil { - panic(err) - } - } - time.Sleep(3 * time.Second) - - // Start injecting transactions from the faucet like crazy - nonces := make([]uint64, len(faucets)) - for { - // Stop when interrupted. - select { - case <-interruptCh: - for _, node := range stacks { - node.Close() - } - return - default: - } - - // Pick a random signer node - index := rand.Intn(len(faucets)) - backend := nodes[index%len(nodes)] - - // Create a self transaction and inject into the pool - tx, err := types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000), nil), types.HomesteadSigner{}, faucets[index]) - if err != nil { - panic(err) - } - if err := backend.TxPool().Add([]*types.Transaction{tx}, true, false); err != nil { - panic(err) - } - nonces[index]++ - - // Wait if we're too saturated - if pend, _ := backend.TxPool().Stats(); pend > 2048 { - time.Sleep(100 * time.Millisecond) - } - } -} - -// makeGenesis creates a custom Clique genesis block based on some pre-defined -// signer and faucet accounts. -func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core.Genesis { - // Create a Clique network based off of the Sepolia config - genesis := core.DefaultSepoliaGenesisBlock() - genesis.GasLimit = 25000000 - - genesis.Config.ChainID = big.NewInt(18) - genesis.Config.Clique.Period = 1 - - genesis.Alloc = types.GenesisAlloc{} - for _, faucet := range faucets { - genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = types.Account{ - Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), - } - } - // Sort the signers and embed into the extra-data section - signers := make([]common.Address, len(sealers)) - for i, sealer := range sealers { - signers[i] = crypto.PubkeyToAddress(sealer.PublicKey) - } - for i := 0; i < len(signers); i++ { - for j := i + 1; j < len(signers); j++ { - if bytes.Compare(signers[i][:], signers[j][:]) > 0 { - signers[i], signers[j] = signers[j], signers[i] - } - } - } - genesis.ExtraData = make([]byte, 32+len(signers)*common.AddressLength+65) - for i, signer := range signers { - copy(genesis.ExtraData[32+i*common.AddressLength:], signer[:]) - } - // Return the genesis block for initialization - return genesis -} - -func makeSealer(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { - // Define the basic configurations for the Ethereum node - datadir, _ := os.MkdirTemp("", "") - - config := &node.Config{ - Name: "geth", - Version: params.Version, - DataDir: datadir, - P2P: p2p.Config{ - ListenAddr: "0.0.0.0:0", - NoDiscovery: true, - MaxPeers: 25, - }, - } - // Start the node and configure a full Ethereum node on it - stack, err := node.New(config) - if err != nil { - return nil, nil, err - } - // Create and register the backend - ethBackend, err := eth.New(stack, ðconfig.Config{ - Genesis: genesis, - NetworkId: genesis.Config.ChainID.Uint64(), - SyncMode: downloader.FullSync, - DatabaseCache: 256, - DatabaseHandles: 256, - TxPool: legacypool.DefaultConfig, - GPO: ethconfig.Defaults.GPO, - Miner: miner.Config{ - GasCeil: genesis.GasLimit * 11 / 10, - GasPrice: big.NewInt(1), - Recommit: time.Second, - }, - }) - if err != nil { - return nil, nil, err - } - - err = stack.Start() - return stack, ethBackend, err -} diff --git a/miner/worker.go b/miner/worker.go index 134f91cafc4b..7e038b0f301b 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -20,12 +20,10 @@ import ( "errors" "fmt" "math/big" - "sync" "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" @@ -33,47 +31,11 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" ) -const ( - // resultQueueSize is the size of channel listening to sealing result. - resultQueueSize = 10 - - // txChanSize is the size of channel listening to NewTxsEvent. - // The number is referenced from the size of tx pool. - txChanSize = 4096 - - // chainHeadChanSize is the size of channel listening to ChainHeadEvent. - chainHeadChanSize = 10 - - // resubmitAdjustChanSize is the size of resubmitting interval adjustment channel. - resubmitAdjustChanSize = 10 - - // minRecommitInterval is the minimal time interval to recreate the sealing block with - // any newly arrived transactions. - minRecommitInterval = 1 * time.Second - - // maxRecommitInterval is the maximum time interval to recreate the sealing block with - // any newly arrived transactions. - maxRecommitInterval = 15 * time.Second - - // intervalAdjustRatio is the impact a single interval adjustment has on sealing work - // resubmitting interval. - intervalAdjustRatio = 0.1 - - // intervalAdjustBias is applied during the new resubmit interval calculation in favor of - // increasing upper limit or decreasing lower limit so that the limit can be reachable. - intervalAdjustBias = 200 * 1000.0 * 1000.0 - - // staleThreshold is the maximum depth of the acceptable stale block. - staleThreshold = 7 -) - var ( errBlockInterruptedByNewHead = errors.New("new head arrived while building block") errBlockInterruptedByRecommit = errors.New("recommit interrupt while building block") @@ -96,47 +58,6 @@ type environment struct { blobs int } -// copy creates a deep copy of environment. -func (env *environment) copy() *environment { - cpy := &environment{ - signer: env.signer, - state: env.state.Copy(), - tcount: env.tcount, - coinbase: env.coinbase, - header: types.CopyHeader(env.header), - receipts: copyReceipts(env.receipts), - } - if env.gasPool != nil { - gasPool := *env.gasPool - cpy.gasPool = &gasPool - } - cpy.txs = make([]*types.Transaction, len(env.txs)) - copy(cpy.txs, env.txs) - - cpy.sidecars = make([]*types.BlobTxSidecar, len(env.sidecars)) - copy(cpy.sidecars, env.sidecars) - - return cpy -} - -// discard terminates the background prefetcher go-routine. It should -// always be called for all created environment instances otherwise -// the go-routine leak can happen. -func (env *environment) discard() { - if env.state == nil { - return - } - env.state.StopPrefetcher() -} - -// task contains all information for consensus engine sealing and result submitting. -type task struct { - receipts []*types.Receipt - state *state.StateDB - block *types.Block - createdAt time.Time -} - const ( commitInterruptNone int32 = iota commitInterruptNewHead @@ -144,629 +65,174 @@ const ( commitInterruptTimeout ) -// newWorkReq represents a request for new sealing work submitting with relative interrupt notifier. -type newWorkReq struct { - interrupt *atomic.Int32 - timestamp int64 -} - // newPayloadResult is the result of payload generation. type newPayloadResult struct { err error block *types.Block fees *big.Int // total block fees sidecars []*types.BlobTxSidecar // collected blobs of blob transactions + stateDB *state.StateDB // StateDB after executing the transactions + receipts []*types.Receipt // Receipts collected during construction } -// getWorkReq represents a request for getting a new sealing work with provided parameters. -type getWorkReq struct { - params *generateParams - result chan *newPayloadResult // non-blocking channel -} - -// intervalAdjust represents a resubmitting interval adjustment. -type intervalAdjust struct { - ratio float64 - inc bool -} - -// worker is the main object which takes care of submitting new work to consensus engine -// and gathering the sealing result. -type worker struct { - config *Config - chainConfig *params.ChainConfig - engine consensus.Engine - eth Backend - chain *core.BlockChain - - // Feeds - pendingLogsFeed event.Feed - - // Subscriptions - mux *event.TypeMux - txsCh chan core.NewTxsEvent - txsSub event.Subscription - chainHeadCh chan core.ChainHeadEvent - chainHeadSub event.Subscription - - // Channels - newWorkCh chan *newWorkReq - getWorkCh chan *getWorkReq - taskCh chan *task - resultCh chan *types.Block - startCh chan struct{} - exitCh chan struct{} - resubmitIntervalCh chan time.Duration - resubmitAdjustCh chan *intervalAdjust - - wg sync.WaitGroup - - current *environment // An environment for current running cycle. - - mu sync.RWMutex // The lock used to protect the coinbase and extra fields - coinbase common.Address - extra []byte - tip *uint256.Int // Minimum tip needed for non-local transaction to include them - - pendingMu sync.RWMutex - pendingTasks map[common.Hash]*task - - snapshotMu sync.RWMutex // The lock used to protect the snapshots below - snapshotBlock *types.Block - snapshotReceipts types.Receipts - snapshotState *state.StateDB - - // atomic status counters - running atomic.Bool // The indicator whether the consensus engine is running or not. - newTxs atomic.Int32 // New arrival transaction count since last sealing work submitting. - syncing atomic.Bool // The indicator whether the node is still syncing. - - // newpayloadTimeout is the maximum timeout allowance for creating payload. - // The default value is 2 seconds but node operator can set it to arbitrary - // large value. A large timeout allowance may cause Geth to fail creating - // a non-empty payload within the specified time and eventually miss the slot - // in case there are some computation expensive transactions in txpool. - newpayloadTimeout time.Duration - - // recommit is the time interval to re-create sealing work or to re-build - // payload in proof-of-stake stage. - recommit time.Duration - - // External functions - isLocalBlock func(header *types.Header) bool // Function used to determine whether the specified block is mined by local miner. - - // Test hooks - newTaskHook func(*task) // Method to call upon receiving a new sealing task. - skipSealHook func(*task) bool // Method to decide whether skipping the sealing. - fullTaskHook func() // Method to call before pushing the full sealing task. - resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. +// generateParams wraps various of settings for generating sealing task. +type generateParams struct { + timestamp uint64 // The timestamp for sealing task + forceTime bool // Flag whether the given timestamp is immutable or not + parentHash common.Hash // Parent block hash, empty means the latest chain head + coinbase common.Address // The fee recipient address for including transaction + random common.Hash // The randomness generated by beacon chain, empty before the merge + withdrawals types.Withdrawals // List of withdrawals to include in block (shanghai field) + beaconRoot *common.Hash // The beacon root (cancun field). + noTxs bool // Flag whether an empty block without any transaction is expected } -func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(header *types.Header) bool, init bool) *worker { - worker := &worker{ - config: config, - chainConfig: chainConfig, - engine: engine, - eth: eth, - chain: eth.BlockChain(), - mux: mux, - isLocalBlock: isLocalBlock, - coinbase: config.Etherbase, - extra: config.ExtraData, - tip: uint256.MustFromBig(config.GasPrice), - pendingTasks: make(map[common.Hash]*task), - txsCh: make(chan core.NewTxsEvent, txChanSize), - chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), - newWorkCh: make(chan *newWorkReq), - getWorkCh: make(chan *getWorkReq), - taskCh: make(chan *task), - resultCh: make(chan *types.Block, resultQueueSize), - startCh: make(chan struct{}, 1), - exitCh: make(chan struct{}), - resubmitIntervalCh: make(chan time.Duration), - resubmitAdjustCh: make(chan *intervalAdjust, resubmitAdjustChanSize), - } - // Subscribe for transaction insertion events (whether from network or resurrects) - worker.txsSub = eth.TxPool().SubscribeTransactions(worker.txsCh, true) - // Subscribe events for blockchain - worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) - - // Sanitize recommit interval if the user-specified one is too short. - recommit := worker.config.Recommit - if recommit < minRecommitInterval { - log.Warn("Sanitizing miner recommit interval", "provided", recommit, "updated", minRecommitInterval) - recommit = minRecommitInterval - } - worker.recommit = recommit - - // Sanitize the timeout config for creating payload. - newpayloadTimeout := worker.config.NewPayloadTimeout - if newpayloadTimeout == 0 { - log.Warn("Sanitizing new payload timeout to default", "provided", newpayloadTimeout, "updated", DefaultConfig.NewPayloadTimeout) - newpayloadTimeout = DefaultConfig.NewPayloadTimeout - } - if newpayloadTimeout < time.Millisecond*100 { - log.Warn("Low payload timeout may cause high amount of non-full blocks", "provided", newpayloadTimeout, "default", DefaultConfig.NewPayloadTimeout) +// generateWork generates a sealing block based on the given parameters. +func (miner *Miner) generateWork(params *generateParams) *newPayloadResult { + work, err := miner.prepareWork(params) + if err != nil { + return &newPayloadResult{err: err} } - worker.newpayloadTimeout = newpayloadTimeout - - worker.wg.Add(4) - go worker.mainLoop() - go worker.newWorkLoop(recommit) - go worker.resultLoop() - go worker.taskLoop() + if !params.noTxs { + interrupt := new(atomic.Int32) + timer := time.AfterFunc(miner.config.Recommit, func() { + interrupt.Store(commitInterruptTimeout) + }) + defer timer.Stop() - // Submit first work to initialize pending state. - if init { - worker.startCh <- struct{}{} + err := miner.fillTransactions(interrupt, work) + if errors.Is(err, errBlockInterruptedByTimeout) { + log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(miner.config.Recommit)) + } } - return worker -} - -// setEtherbase sets the etherbase used to initialize the block coinbase field. -func (w *worker) setEtherbase(addr common.Address) { - w.mu.Lock() - defer w.mu.Unlock() - w.coinbase = addr -} - -// etherbase retrieves the configured etherbase address. -func (w *worker) etherbase() common.Address { - w.mu.RLock() - defer w.mu.RUnlock() - return w.coinbase -} - -func (w *worker) setGasCeil(ceil uint64) { - w.mu.Lock() - defer w.mu.Unlock() - w.config.GasCeil = ceil -} - -// setExtra sets the content used to initialize the block extra field. -func (w *worker) setExtra(extra []byte) { - w.mu.Lock() - defer w.mu.Unlock() - w.extra = extra -} - -// setGasTip sets the minimum miner tip needed to include a non-local transaction. -func (w *worker) setGasTip(tip *big.Int) { - w.mu.Lock() - defer w.mu.Unlock() - w.tip = uint256.MustFromBig(tip) -} - -// setRecommitInterval updates the interval for miner sealing work recommitting. -func (w *worker) setRecommitInterval(interval time.Duration) { - select { - case w.resubmitIntervalCh <- interval: - case <-w.exitCh: + block, err := miner.engine.FinalizeAndAssemble(miner.chain, work.header, work.state, work.txs, nil, work.receipts, params.withdrawals) + if err != nil { + return &newPayloadResult{err: err} } -} - -// pending returns the pending state and corresponding block. The returned -// values can be nil in case the pending block is not initialized. -func (w *worker) pending() (*types.Block, *state.StateDB) { - w.snapshotMu.RLock() - defer w.snapshotMu.RUnlock() - if w.snapshotState == nil { - return nil, nil + return &newPayloadResult{ + block: block, + fees: totalFees(block, work.receipts), + sidecars: work.sidecars, + stateDB: work.state, + receipts: work.receipts, } - return w.snapshotBlock, w.snapshotState.Copy() } -// pendingBlock returns pending block. The returned block can be nil in case the -// pending block is not initialized. -func (w *worker) pendingBlock() *types.Block { - w.snapshotMu.RLock() - defer w.snapshotMu.RUnlock() - return w.snapshotBlock -} - -// pendingBlockAndReceipts returns pending block and corresponding receipts. -// The returned values can be nil in case the pending block is not initialized. -func (w *worker) pendingBlockAndReceipts() (*types.Block, types.Receipts) { - w.snapshotMu.RLock() - defer w.snapshotMu.RUnlock() - return w.snapshotBlock, w.snapshotReceipts -} - -// start sets the running status as 1 and triggers new work submitting. -func (w *worker) start() { - w.running.Store(true) - w.startCh <- struct{}{} -} - -// stop sets the running status as 0. -func (w *worker) stop() { - w.running.Store(false) -} - -// isRunning returns an indicator whether worker is running or not. -func (w *worker) isRunning() bool { - return w.running.Load() -} - -// close terminates all background threads maintained by the worker. -// Note the worker does not support being closed multiple times. -func (w *worker) close() { - w.running.Store(false) - close(w.exitCh) - w.wg.Wait() -} +// prepareWork constructs the sealing task according to the given parameters, +// either based on the last chain head or specified parent. In this function +// the pending transactions are not filled yet, only the empty task returned. +func (miner *Miner) prepareWork(genParams *generateParams) (*environment, error) { + miner.confMu.RLock() + defer miner.confMu.RUnlock() -// recalcRecommit recalculates the resubmitting interval upon feedback. -func recalcRecommit(minRecommit, prev time.Duration, target float64, inc bool) time.Duration { - var ( - prevF = float64(prev.Nanoseconds()) - next float64 - ) - if inc { - next = prevF*(1-intervalAdjustRatio) + intervalAdjustRatio*(target+intervalAdjustBias) - max := float64(maxRecommitInterval.Nanoseconds()) - if next > max { - next = max - } - } else { - next = prevF*(1-intervalAdjustRatio) + intervalAdjustRatio*(target-intervalAdjustBias) - min := float64(minRecommit.Nanoseconds()) - if next < min { - next = min + // Find the parent block for sealing task + parent := miner.chain.CurrentBlock() + if genParams.parentHash != (common.Hash{}) { + block := miner.chain.GetBlockByHash(genParams.parentHash) + if block == nil { + return nil, fmt.Errorf("missing parent") } + parent = block.Header() } - return time.Duration(int64(next)) -} - -// newWorkLoop is a standalone goroutine to submit new sealing work upon received events. -func (w *worker) newWorkLoop(recommit time.Duration) { - defer w.wg.Done() - var ( - interrupt *atomic.Int32 - minRecommit = recommit // minimal resubmit interval specified by user. - timestamp int64 // timestamp for each round of sealing. - ) - - timer := time.NewTimer(0) - defer timer.Stop() - <-timer.C // discard the initial tick - - // commit aborts in-flight transaction execution with given signal and resubmits a new one. - commit := func(s int32) { - if interrupt != nil { - interrupt.Store(s) - } - interrupt = new(atomic.Int32) - select { - case w.newWorkCh <- &newWorkReq{interrupt: interrupt, timestamp: timestamp}: - case <-w.exitCh: - return + // Sanity check the timestamp correctness, recap the timestamp + // to parent+1 if the mutation is allowed. + timestamp := genParams.timestamp + if parent.Time >= timestamp { + if genParams.forceTime { + return nil, fmt.Errorf("invalid timestamp, parent %d given %d", parent.Time, timestamp) } - timer.Reset(recommit) - w.newTxs.Store(0) + timestamp = parent.Time + 1 } - // clearPending cleans the stale pending tasks. - clearPending := func(number uint64) { - w.pendingMu.Lock() - for h, t := range w.pendingTasks { - if t.block.NumberU64()+staleThreshold <= number { - delete(w.pendingTasks, h) - } - } - w.pendingMu.Unlock() + // Construct the sealing block header. + header := &types.Header{ + ParentHash: parent.Hash(), + Number: new(big.Int).Add(parent.Number, common.Big1), + GasLimit: core.CalcGasLimit(parent.GasLimit, miner.config.GasCeil), + Time: timestamp, + Coinbase: genParams.coinbase, } - - for { - select { - case <-w.startCh: - clearPending(w.chain.CurrentBlock().Number.Uint64()) - timestamp = time.Now().Unix() - commit(commitInterruptNewHead) - - case head := <-w.chainHeadCh: - clearPending(head.Block.NumberU64()) - timestamp = time.Now().Unix() - commit(commitInterruptNewHead) - - case <-timer.C: - // If sealing is running resubmit a new work cycle periodically to pull in - // higher priced transactions. Disable this overhead for pending blocks. - if w.isRunning() && (w.chainConfig.Clique == nil || w.chainConfig.Clique.Period > 0) { - // Short circuit if no new transaction arrives. - if w.newTxs.Load() == 0 { - timer.Reset(recommit) - continue - } - commit(commitInterruptResubmit) - } - - case interval := <-w.resubmitIntervalCh: - // Adjust resubmit interval explicitly by user. - if interval < minRecommitInterval { - log.Warn("Sanitizing miner recommit interval", "provided", interval, "updated", minRecommitInterval) - interval = minRecommitInterval - } - log.Info("Miner recommit interval update", "from", minRecommit, "to", interval) - minRecommit, recommit = interval, interval - - if w.resubmitHook != nil { - w.resubmitHook(minRecommit, recommit) - } - - case adjust := <-w.resubmitAdjustCh: - // Adjust resubmit interval by feedback. - if adjust.inc { - before := recommit - target := float64(recommit.Nanoseconds()) / adjust.ratio - recommit = recalcRecommit(minRecommit, recommit, target, true) - log.Trace("Increase miner recommit interval", "from", before, "to", recommit) - } else { - before := recommit - recommit = recalcRecommit(minRecommit, recommit, float64(minRecommit.Nanoseconds()), false) - log.Trace("Decrease miner recommit interval", "from", before, "to", recommit) - } - - if w.resubmitHook != nil { - w.resubmitHook(minRecommit, recommit) - } - - case <-w.exitCh: - return - } + // Set the extra field. + if len(miner.config.ExtraData) != 0 { + header.Extra = miner.config.ExtraData } -} - -// mainLoop is responsible for generating and submitting sealing work based on -// the received event. It can support two modes: automatically generate task and -// submit it or return task according to given parameters for various proposes. -func (w *worker) mainLoop() { - defer w.wg.Done() - defer w.txsSub.Unsubscribe() - defer w.chainHeadSub.Unsubscribe() - defer func() { - if w.current != nil { - w.current.discard() - } - }() - - for { - select { - case req := <-w.newWorkCh: - w.commitWork(req.interrupt, req.timestamp) - - case req := <-w.getWorkCh: - req.result <- w.generateWork(req.params) - - case ev := <-w.txsCh: - // Apply transactions to the pending state if we're not sealing - // - // Note all transactions received may not be continuous with transactions - // already included in the current sealing block. These transactions will - // be automatically eliminated. - if !w.isRunning() && w.current != nil { - // If block is already full, abort - if gp := w.current.gasPool; gp != nil && gp.Gas() < params.TxGas { - continue - } - txs := make(map[common.Address][]*txpool.LazyTransaction, len(ev.Txs)) - for _, tx := range ev.Txs { - acc, _ := types.Sender(w.current.signer, tx) - txs[acc] = append(txs[acc], &txpool.LazyTransaction{ - Pool: w.eth.TxPool(), // We don't know where this came from, yolo resolve from everywhere - Hash: tx.Hash(), - Tx: nil, // Do *not* set this! We need to resolve it later to pull blobs in - Time: tx.Time(), - GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), - GasTipCap: uint256.MustFromBig(tx.GasTipCap()), - Gas: tx.Gas(), - BlobGas: tx.BlobGas(), - }) - } - plainTxs := newTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) // Mixed bag of everrything, yolo - blobTxs := newTransactionsByPriceAndNonce(w.current.signer, nil, w.current.header.BaseFee) // Empty bag, don't bother optimising - - tcount := w.current.tcount - w.commitTransactions(w.current, plainTxs, blobTxs, nil) - - // Only update the snapshot if any new transactions were added - // to the pending block - if tcount != w.current.tcount { - w.updateSnapshot(w.current) - } - } else { - // Special case, if the consensus engine is 0 period clique(dev mode), - // submit sealing work here since all empty submission will be rejected - // by clique. Of course the advance sealing(empty submission) is disabled. - if w.chainConfig.Clique != nil && w.chainConfig.Clique.Period == 0 { - w.commitWork(nil, time.Now().Unix()) - } - } - w.newTxs.Add(int32(len(ev.Txs))) - - // System stopped - case <-w.exitCh: - return - case <-w.txsSub.Err(): - return - case <-w.chainHeadSub.Err(): - return - } + // Set the randomness field from the beacon chain if it's available. + if genParams.random != (common.Hash{}) { + header.MixDigest = genParams.random } -} - -// taskLoop is a standalone goroutine to fetch sealing task from the generator and -// push them to consensus engine. -func (w *worker) taskLoop() { - defer w.wg.Done() - var ( - stopCh chan struct{} - prev common.Hash - ) - - // interrupt aborts the in-flight sealing task. - interrupt := func() { - if stopCh != nil { - close(stopCh) - stopCh = nil + // Set baseFee and GasLimit if we are on an EIP-1559 chain + if miner.chainConfig.IsLondon(header.Number) { + header.BaseFee = eip1559.CalcBaseFee(miner.chainConfig, parent) + if !miner.chainConfig.IsLondon(parent.Number) { + parentGasLimit := parent.GasLimit * miner.chainConfig.ElasticityMultiplier() + header.GasLimit = core.CalcGasLimit(parentGasLimit, miner.config.GasCeil) } } - for { - select { - case task := <-w.taskCh: - if w.newTaskHook != nil { - w.newTaskHook(task) - } - // Reject duplicate sealing work due to resubmitting. - sealHash := w.engine.SealHash(task.block.Header()) - if sealHash == prev { - continue - } - // Interrupt previous sealing operation - interrupt() - stopCh, prev = make(chan struct{}), sealHash - - if w.skipSealHook != nil && w.skipSealHook(task) { - continue - } - w.pendingMu.Lock() - w.pendingTasks[sealHash] = task - w.pendingMu.Unlock() - - if err := w.engine.Seal(w.chain, task.block, w.resultCh, stopCh); err != nil { - log.Warn("Block sealing failed", "err", err) - w.pendingMu.Lock() - delete(w.pendingTasks, sealHash) - w.pendingMu.Unlock() - } - case <-w.exitCh: - interrupt() - return + // Apply EIP-4844, EIP-4788. + if miner.chainConfig.IsCancun(header.Number, header.Time) { + var excessBlobGas uint64 + if miner.chainConfig.IsCancun(parent.Number, parent.Time) { + excessBlobGas = eip4844.CalcExcessBlobGas(*parent.ExcessBlobGas, *parent.BlobGasUsed) + } else { + // For the first post-fork block, both parent.data_gas_used and parent.excess_data_gas are evaluated as 0 + excessBlobGas = eip4844.CalcExcessBlobGas(0, 0) } + header.BlobGasUsed = new(uint64) + header.ExcessBlobGas = &excessBlobGas + header.ParentBeaconRoot = genParams.beaconRoot } -} - -// resultLoop is a standalone goroutine to handle sealing result submitting -// and flush relative data to the database. -func (w *worker) resultLoop() { - defer w.wg.Done() - for { - select { - case block := <-w.resultCh: - // Short circuit when receiving empty result. - if block == nil { - continue - } - // Short circuit when receiving duplicate result caused by resubmitting. - if w.chain.HasBlock(block.Hash(), block.NumberU64()) { - continue - } - var ( - sealhash = w.engine.SealHash(block.Header()) - hash = block.Hash() - ) - w.pendingMu.RLock() - task, exist := w.pendingTasks[sealhash] - w.pendingMu.RUnlock() - if !exist { - log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", hash) - continue - } - // Different block could share same sealhash, deep copy here to prevent write-write conflict. - var ( - receipts = make([]*types.Receipt, len(task.receipts)) - logs []*types.Log - ) - for i, taskReceipt := range task.receipts { - receipt := new(types.Receipt) - receipts[i] = receipt - *receipt = *taskReceipt - - // add block location fields - receipt.BlockHash = hash - receipt.BlockNumber = block.Number() - receipt.TransactionIndex = uint(i) - - // Update the block hash in all logs since it is now available and not when the - // receipt/log of individual transactions were created. - receipt.Logs = make([]*types.Log, len(taskReceipt.Logs)) - for i, taskLog := range taskReceipt.Logs { - log := new(types.Log) - receipt.Logs[i] = log - *log = *taskLog - log.BlockHash = hash - } - logs = append(logs, receipt.Logs...) - } - // Commit block and state to database. - _, err := w.chain.WriteBlockAndSetHead(block, receipts, logs, task.state, true) - if err != nil { - log.Error("Failed writing block to chain", "err", err) - continue - } - log.Info("Successfully sealed new block", "number", block.Number(), "sealhash", sealhash, "hash", hash, - "elapsed", common.PrettyDuration(time.Since(task.createdAt))) - - // Broadcast the block and announce chain insertion event - w.mux.Post(core.NewMinedBlockEvent{Block: block}) - - case <-w.exitCh: - return - } + // Run the consensus preparation with the default or customized consensus engine. + if err := miner.engine.Prepare(miner.chain, header); err != nil { + log.Error("Failed to prepare header for sealing", "err", err) + return nil, err + } + // Could potentially happen if starting to mine in an odd state. + // Note genParams.coinbase can be different with header.Coinbase + // since clique algorithm can modify the coinbase field in header. + env, err := miner.makeEnv(parent, header, genParams.coinbase) + if err != nil { + log.Error("Failed to create sealing context", "err", err) + return nil, err + } + if header.ParentBeaconRoot != nil { + context := core.NewEVMBlockContext(header, miner.chain, nil) + vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, miner.chainConfig, vm.Config{}) + core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, vmenv, env.state) } + return env, nil } // makeEnv creates a new environment for the sealing block. -func (w *worker) makeEnv(parent *types.Header, header *types.Header, coinbase common.Address) (*environment, error) { +func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase common.Address) (*environment, error) { // Retrieve the parent state to execute on top and start a prefetcher for // the miner to speed block sealing up a bit. - state, err := w.chain.StateAt(parent.Root) + state, err := miner.chain.StateAt(parent.Root) if err != nil { return nil, err } - state.StartPrefetcher("miner") - // Note the passed coinbase may be different with header.Coinbase. - env := &environment{ - signer: types.MakeSigner(w.chainConfig, header.Number, header.Time), + return &environment{ + signer: types.MakeSigner(miner.chainConfig, header.Number, header.Time), state: state, coinbase: coinbase, header: header, - } - // Keep track of transactions which return errors so they can be removed - env.tcount = 0 - return env, nil + }, nil } -// updateSnapshot updates pending snapshot block, receipts and state. -func (w *worker) updateSnapshot(env *environment) { - w.snapshotMu.Lock() - defer w.snapshotMu.Unlock() - - w.snapshotBlock = types.NewBlock( - env.header, - env.txs, - nil, - env.receipts, - trie.NewStackTrie(nil), - ) - w.snapshotReceipts = copyReceipts(env.receipts) - w.snapshotState = env.state.Copy() -} - -func (w *worker) commitTransaction(env *environment, tx *types.Transaction) ([]*types.Log, error) { +func (miner *Miner) commitTransaction(env *environment, tx *types.Transaction) error { if tx.Type() == types.BlobTxType { - return w.commitBlobTransaction(env, tx) + return miner.commitBlobTransaction(env, tx) } - receipt, err := w.applyTransaction(env, tx) + receipt, err := miner.applyTransaction(env, tx) if err != nil { - return nil, err + return err } env.txs = append(env.txs, tx) env.receipts = append(env.receipts, receipt) - return receipt.Logs, nil + env.tcount++ + return nil } -func (w *worker) commitBlobTransaction(env *environment, tx *types.Transaction) ([]*types.Log, error) { +func (miner *Miner) commitBlobTransaction(env *environment, tx *types.Transaction) error { sc := tx.BlobTxSidecar() if sc == nil { panic("blob transaction without blobs in miner") @@ -776,27 +242,28 @@ func (w *worker) commitBlobTransaction(env *environment, tx *types.Transaction) // and not during execution. This means core.ApplyTransaction will not return an error if the // tx has too many blobs. So we have to explicitly check it here. if (env.blobs+len(sc.Blobs))*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { - return nil, errors.New("max data blobs reached") + return errors.New("max data blobs reached") } - receipt, err := w.applyTransaction(env, tx) + receipt, err := miner.applyTransaction(env, tx) if err != nil { - return nil, err + return err } env.txs = append(env.txs, tx.WithoutBlobTxSidecar()) env.receipts = append(env.receipts, receipt) env.sidecars = append(env.sidecars, sc) env.blobs += len(sc.Blobs) *env.header.BlobGasUsed += receipt.BlobGasUsed - return receipt.Logs, nil + env.tcount++ + return nil } // applyTransaction runs the transaction. If execution fails, state and gas pool are reverted. -func (w *worker) applyTransaction(env *environment, tx *types.Transaction) (*types.Receipt, error) { +func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (*types.Receipt, error) { var ( snap = env.state.Snapshot() gp = env.gasPool.Gas() ) - receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &env.coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, *w.chain.GetVMConfig()) + receipt, err := core.ApplyTransaction(miner.chainConfig, miner.chain, &env.coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, *miner.chain.GetVMConfig()) if err != nil { env.state.RevertToSnapshot(snap) env.gasPool.SetGas(gp) @@ -804,13 +271,11 @@ func (w *worker) applyTransaction(env *environment, tx *types.Transaction) (*typ return receipt, err } -func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { +func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { gasLimit := env.header.GasLimit if env.gasPool == nil { env.gasPool = new(core.GasPool).AddGas(gasLimit) } - var coalescedLogs []*types.Log - for { // Check interruption signal and abort building if it's fired. if interrupt != nil { @@ -877,15 +342,15 @@ func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transac // Check whether the tx is replay protected. If we're not in the EIP155 hf // phase, start ignoring the sender until we do. - if tx.Protected() && !w.chainConfig.IsEIP155(env.header.Number) { - log.Trace("Ignoring replay protected transaction", "hash", ltx.Hash, "eip155", w.chainConfig.EIP155Block) + if tx.Protected() && !miner.chainConfig.IsEIP155(env.header.Number) { + log.Trace("Ignoring replay protected transaction", "hash", ltx.Hash, "eip155", miner.chainConfig.EIP155Block) txs.Pop() continue } // Start executing the transaction env.state.SetTxContext(tx.Hash(), env.tcount) - logs, err := w.commitTransaction(env, tx) + err := miner.commitTransaction(env, tx) switch { case errors.Is(err, core.ErrNonceTooLow): // New head notification data race between the transaction pool and miner, shift @@ -894,8 +359,6 @@ func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transac case errors.Is(err, nil): // Everything ok, collect the logs and shift in the next transaction from the same account - coalescedLogs = append(coalescedLogs, logs...) - env.tcount++ txs.Shift() default: @@ -905,130 +368,20 @@ func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transac txs.Pop() } } - if !w.isRunning() && len(coalescedLogs) > 0 { - // We don't push the pendingLogsEvent while we are sealing. The reason is that - // when we are sealing, the worker will regenerate a sealing block every 3 seconds. - // In order to avoid pushing the repeated pendingLog, we disable the pending log pushing. - - // make a copy, the state caches the logs and these logs get "upgraded" from pending to mined - // logs by filling in the block hash when the block was mined by the local miner. This can - // cause a race condition if a log was "upgraded" before the PendingLogsEvent is processed. - cpy := make([]*types.Log, len(coalescedLogs)) - for i, l := range coalescedLogs { - cpy[i] = new(types.Log) - *cpy[i] = *l - } - w.pendingLogsFeed.Send(cpy) - } return nil } -// generateParams wraps various of settings for generating sealing task. -type generateParams struct { - timestamp uint64 // The timestamp for sealing task - forceTime bool // Flag whether the given timestamp is immutable or not - parentHash common.Hash // Parent block hash, empty means the latest chain head - coinbase common.Address // The fee recipient address for including transaction - random common.Hash // The randomness generated by beacon chain, empty before the merge - withdrawals types.Withdrawals // List of withdrawals to include in block. - beaconRoot *common.Hash // The beacon root (cancun field). - noTxs bool // Flag whether an empty block without any transaction is expected -} - -// prepareWork constructs the sealing task according to the given parameters, -// either based on the last chain head or specified parent. In this function -// the pending transactions are not filled yet, only the empty task returned. -func (w *worker) prepareWork(genParams *generateParams) (*environment, error) { - w.mu.RLock() - defer w.mu.RUnlock() - - // Find the parent block for sealing task - parent := w.chain.CurrentBlock() - if genParams.parentHash != (common.Hash{}) { - block := w.chain.GetBlockByHash(genParams.parentHash) - if block == nil { - return nil, errors.New("missing parent") - } - parent = block.Header() - } - // Sanity check the timestamp correctness, recap the timestamp - // to parent+1 if the mutation is allowed. - timestamp := genParams.timestamp - if parent.Time >= timestamp { - if genParams.forceTime { - return nil, fmt.Errorf("invalid timestamp, parent %d given %d", parent.Time, timestamp) - } - timestamp = parent.Time + 1 - } - // Construct the sealing block header. - header := &types.Header{ - ParentHash: parent.Hash(), - Number: new(big.Int).Add(parent.Number, common.Big1), - GasLimit: core.CalcGasLimit(parent.GasLimit, w.config.GasCeil), - Time: timestamp, - Coinbase: genParams.coinbase, - } - // Set the extra field. - if len(w.extra) != 0 { - header.Extra = w.extra - } - // Set the randomness field from the beacon chain if it's available. - if genParams.random != (common.Hash{}) { - header.MixDigest = genParams.random - } - // Set baseFee and GasLimit if we are on an EIP-1559 chain - if w.chainConfig.IsLondon(header.Number) { - header.BaseFee = eip1559.CalcBaseFee(w.chainConfig, parent) - if !w.chainConfig.IsLondon(parent.Number) { - parentGasLimit := parent.GasLimit * w.chainConfig.ElasticityMultiplier() - header.GasLimit = core.CalcGasLimit(parentGasLimit, w.config.GasCeil) - } - } - // Apply EIP-4844, EIP-4788. - if w.chainConfig.IsCancun(header.Number, header.Time) { - var excessBlobGas uint64 - if w.chainConfig.IsCancun(parent.Number, parent.Time) { - excessBlobGas = eip4844.CalcExcessBlobGas(*parent.ExcessBlobGas, *parent.BlobGasUsed) - } else { - // For the first post-fork block, both parent.data_gas_used and parent.excess_data_gas are evaluated as 0 - excessBlobGas = eip4844.CalcExcessBlobGas(0, 0) - } - header.BlobGasUsed = new(uint64) - header.ExcessBlobGas = &excessBlobGas - header.ParentBeaconRoot = genParams.beaconRoot - } - // Run the consensus preparation with the default or customized consensus engine. - if err := w.engine.Prepare(w.chain, header); err != nil { - log.Error("Failed to prepare header for sealing", "err", err) - return nil, err - } - // Could potentially happen if starting to mine in an odd state. - // Note genParams.coinbase can be different with header.Coinbase - // since clique algorithm can modify the coinbase field in header. - env, err := w.makeEnv(parent, header, genParams.coinbase) - if err != nil { - log.Error("Failed to create sealing context", "err", err) - return nil, err - } - if header.ParentBeaconRoot != nil { - context := core.NewEVMBlockContext(header, w.chain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, w.chainConfig, vm.Config{}) - core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, vmenv, env.state) - } - return env, nil -} - // fillTransactions retrieves the pending transactions from the txpool and fills them // into the given sealing block. The transaction selection and ordering strategy can // be customized with the plugin in the future. -func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) error { - w.mu.RLock() - tip := w.tip - w.mu.RUnlock() +func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) error { + miner.confMu.RLock() + tip := miner.config.GasPrice + miner.confMu.RUnlock() // Retrieve the pending transactions pre-filtered by the 1559/4844 dynamic fees filter := txpool.PendingFilter{ - MinTip: tip, + MinTip: uint256.MustFromBig(tip), } if env.header.BaseFee != nil { filter.BaseFee = uint256.MustFromBig(env.header.BaseFee) @@ -1037,16 +390,16 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err filter.BlobFee = uint256.MustFromBig(eip4844.CalcBlobFee(*env.header.ExcessBlobGas)) } filter.OnlyPlainTxs, filter.OnlyBlobTxs = true, false - pendingPlainTxs := w.eth.TxPool().Pending(filter) + pendingPlainTxs := miner.txpool.Pending(filter) filter.OnlyPlainTxs, filter.OnlyBlobTxs = false, true - pendingBlobTxs := w.eth.TxPool().Pending(filter) + pendingBlobTxs := miner.txpool.Pending(filter) // Split the pending transactions into locals and remotes. localPlainTxs, remotePlainTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingPlainTxs localBlobTxs, remoteBlobTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingBlobTxs - for _, account := range w.eth.TxPool().Locals() { + for _, account := range miner.txpool.Locals() { if txs := remotePlainTxs[account]; len(txs) > 0 { delete(remotePlainTxs, account) localPlainTxs[account] = txs @@ -1061,7 +414,7 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err plainTxs := newTransactionsByPriceAndNonce(env.signer, localPlainTxs, env.header.BaseFee) blobTxs := newTransactionsByPriceAndNonce(env.signer, localBlobTxs, env.header.BaseFee) - if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { + if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } @@ -1069,189 +422,13 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err plainTxs := newTransactionsByPriceAndNonce(env.signer, remotePlainTxs, env.header.BaseFee) blobTxs := newTransactionsByPriceAndNonce(env.signer, remoteBlobTxs, env.header.BaseFee) - if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { + if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } return nil } -// generateWork generates a sealing block based on the given parameters. -func (w *worker) generateWork(params *generateParams) *newPayloadResult { - work, err := w.prepareWork(params) - if err != nil { - return &newPayloadResult{err: err} - } - defer work.discard() - - if !params.noTxs { - interrupt := new(atomic.Int32) - timer := time.AfterFunc(w.newpayloadTimeout, func() { - interrupt.Store(commitInterruptTimeout) - }) - defer timer.Stop() - - err := w.fillTransactions(interrupt, work) - if errors.Is(err, errBlockInterruptedByTimeout) { - log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(w.newpayloadTimeout)) - } - } - block, err := w.engine.FinalizeAndAssemble(w.chain, work.header, work.state, work.txs, nil, work.receipts, params.withdrawals) - if err != nil { - return &newPayloadResult{err: err} - } - return &newPayloadResult{ - block: block, - fees: totalFees(block, work.receipts), - sidecars: work.sidecars, - } -} - -// commitWork generates several new sealing tasks based on the parent block -// and submit them to the sealer. -func (w *worker) commitWork(interrupt *atomic.Int32, timestamp int64) { - // Abort committing if node is still syncing - if w.syncing.Load() { - return - } - start := time.Now() - - // Set the coinbase if the worker is running or it's required - var coinbase common.Address - if w.isRunning() { - coinbase = w.etherbase() - if coinbase == (common.Address{}) { - log.Error("Refusing to mine without etherbase") - return - } - } - work, err := w.prepareWork(&generateParams{ - timestamp: uint64(timestamp), - coinbase: coinbase, - }) - if err != nil { - return - } - // Fill pending transactions from the txpool into the block. - err = w.fillTransactions(interrupt, work) - switch { - case err == nil: - // The entire block is filled, decrease resubmit interval in case - // of current interval is larger than the user-specified one. - w.adjustResubmitInterval(&intervalAdjust{inc: false}) - - case errors.Is(err, errBlockInterruptedByRecommit): - // Notify resubmit loop to increase resubmitting interval if the - // interruption is due to frequent commits. - gaslimit := work.header.GasLimit - ratio := float64(gaslimit-work.gasPool.Gas()) / float64(gaslimit) - if ratio < 0.1 { - ratio = 0.1 - } - w.adjustResubmitInterval(&intervalAdjust{ - ratio: ratio, - inc: true, - }) - - case errors.Is(err, errBlockInterruptedByNewHead): - // If the block building is interrupted by newhead event, discard it - // totally. Committing the interrupted block introduces unnecessary - // delay, and possibly causes miner to mine on the previous head, - // which could result in higher uncle rate. - work.discard() - return - } - // Submit the generated block for consensus sealing. - w.commit(work.copy(), w.fullTaskHook, true, start) - - // Swap out the old work with the new one, terminating any leftover - // prefetcher processes in the mean time and starting a new one. - if w.current != nil { - w.current.discard() - } - w.current = work -} - -// commit runs any post-transaction state modifications, assembles the final block -// and commits new work if consensus engine is running. -// Note the assumption is held that the mutation is allowed to the passed env, do -// the deep copy first. -func (w *worker) commit(env *environment, interval func(), update bool, start time.Time) error { - if w.isRunning() { - if interval != nil { - interval() - } - // Create a local environment copy, avoid the data race with snapshot state. - // https://github.com/ethereum/go-ethereum/issues/24299 - env := env.copy() - // Withdrawals are set to nil here, because this is only called in PoW. - block, err := w.engine.FinalizeAndAssemble(w.chain, env.header, env.state, env.txs, nil, env.receipts, nil) - if err != nil { - return err - } - // If we're post merge, just ignore - if !w.isTTDReached(block.Header()) { - select { - case w.taskCh <- &task{receipts: env.receipts, state: env.state, block: block, createdAt: time.Now()}: - fees := totalFees(block, env.receipts) - feesInEther := new(big.Float).Quo(new(big.Float).SetInt(fees), big.NewFloat(params.Ether)) - log.Info("Commit new sealing work", "number", block.Number(), "sealhash", w.engine.SealHash(block.Header()), - "txs", env.tcount, "gas", block.GasUsed(), "fees", feesInEther, - "elapsed", common.PrettyDuration(time.Since(start))) - - case <-w.exitCh: - log.Info("Worker has exited") - } - } - } - if update { - w.updateSnapshot(env) - } - return nil -} - -// getSealingBlock generates the sealing block based on the given parameters. -// The generation result will be passed back via the given channel no matter -// the generation itself succeeds or not. -func (w *worker) getSealingBlock(params *generateParams) *newPayloadResult { - req := &getWorkReq{ - params: params, - result: make(chan *newPayloadResult, 1), - } - select { - case w.getWorkCh <- req: - return <-req.result - case <-w.exitCh: - return &newPayloadResult{err: errors.New("miner closed")} - } -} - -// isTTDReached returns the indicator if the given block has reached the total -// terminal difficulty for The Merge transition. -func (w *worker) isTTDReached(header *types.Header) bool { - td, ttd := w.chain.GetTd(header.ParentHash, header.Number.Uint64()-1), w.chain.Config().TerminalTotalDifficulty - return td != nil && ttd != nil && td.Cmp(ttd) >= 0 -} - -// adjustResubmitInterval adjusts the resubmit interval. -func (w *worker) adjustResubmitInterval(message *intervalAdjust) { - select { - case w.resubmitAdjustCh <- message: - default: - log.Warn("the resubmitAdjustCh is full, discard the message") - } -} - -// copyReceipts makes a deep copy of the given receipts. -func copyReceipts(receipts []*types.Receipt) []*types.Receipt { - result := make([]*types.Receipt, len(receipts)) - for i, l := range receipts { - cpy := *l - result[i] = &cpy - } - return result -} - // totalFees computes total consumed miner fees in Wei. Block transactions and receipts have to have the same order. func totalFees(block *types.Block, receipts []*types.Receipt) *big.Int { feesWei := new(big.Int) diff --git a/miner/worker_test.go b/miner/worker_test.go deleted file mode 100644 index 9dba12ae51a2..000000000000 --- a/miner/worker_test.go +++ /dev/null @@ -1,510 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package miner - -import ( - "math/big" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/clique" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/txpool" - "github.com/ethereum/go-ethereum/core/txpool/legacypool" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/params" - "github.com/holiman/uint256" -) - -const ( - // testCode is the testing contract binary code which will initialises some - // variables in constructor - testCode = "0x60806040527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0060005534801561003457600080fd5b5060fc806100436000396000f3fe6080604052348015600f57600080fd5b506004361060325760003560e01c80630c4dae8814603757806398a213cf146053575b600080fd5b603d607e565b6040518082815260200191505060405180910390f35b607c60048036036020811015606757600080fd5b81019080803590602001909291905050506084565b005b60005481565b806000819055507fe9e44f9f7da8c559de847a3232b57364adc0354f15a2cd8dc636d54396f9587a6000546040518082815260200191505060405180910390a15056fea265627a7a723058208ae31d9424f2d0bc2a3da1a5dd659db2d71ec322a17db8f87e19e209e3a1ff4a64736f6c634300050a0032" - - // testGas is the gas required for contract deployment. - testGas = 144109 -) - -var ( - // Test chain configurations - testTxPoolConfig legacypool.Config - ethashChainConfig *params.ChainConfig - cliqueChainConfig *params.ChainConfig - - // Test accounts - testBankKey, _ = crypto.GenerateKey() - testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) - testBankFunds = big.NewInt(1000000000000000000) - - testUserKey, _ = crypto.GenerateKey() - testUserAddress = crypto.PubkeyToAddress(testUserKey.PublicKey) - - // Test transactions - pendingTxs []*types.Transaction - newTxs []*types.Transaction - - testConfig = &Config{ - Recommit: time.Second, - GasCeil: params.GenesisGasLimit, - } -) - -func init() { - testTxPoolConfig = legacypool.DefaultConfig - testTxPoolConfig.Journal = "" - ethashChainConfig = new(params.ChainConfig) - *ethashChainConfig = *params.TestChainConfig - cliqueChainConfig = new(params.ChainConfig) - *cliqueChainConfig = *params.TestChainConfig - cliqueChainConfig.Clique = ¶ms.CliqueConfig{ - Period: 10, - Epoch: 30000, - } - - signer := types.LatestSigner(params.TestChainConfig) - tx1 := types.MustSignNewTx(testBankKey, signer, &types.AccessListTx{ - ChainID: params.TestChainConfig.ChainID, - Nonce: 0, - To: &testUserAddress, - Value: big.NewInt(1000), - Gas: params.TxGas, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - pendingTxs = append(pendingTxs, tx1) - - tx2 := types.MustSignNewTx(testBankKey, signer, &types.LegacyTx{ - Nonce: 1, - To: &testUserAddress, - Value: big.NewInt(1000), - Gas: params.TxGas, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - newTxs = append(newTxs, tx2) -} - -// testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing. -type testWorkerBackend struct { - db ethdb.Database - txPool *txpool.TxPool - chain *core.BlockChain - genesis *core.Genesis -} - -func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, n int) *testWorkerBackend { - var gspec = &core.Genesis{ - Config: chainConfig, - Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, - } - switch e := engine.(type) { - case *clique.Clique: - gspec.ExtraData = make([]byte, 32+common.AddressLength+crypto.SignatureLength) - copy(gspec.ExtraData[32:32+common.AddressLength], testBankAddress.Bytes()) - e.Authorize(testBankAddress, func(account accounts.Account, s string, data []byte) ([]byte, error) { - return crypto.Sign(crypto.Keccak256(data), testBankKey) - }) - case *ethash.Ethash: - default: - t.Fatalf("unexpected consensus engine type: %T", engine) - } - chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec, nil, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("core.NewBlockChain failed: %v", err) - } - pool := legacypool.New(testTxPoolConfig, chain) - txpool, _ := txpool.New(testTxPoolConfig.PriceLimit, chain, []txpool.SubPool{pool}) - - return &testWorkerBackend{ - db: db, - chain: chain, - txPool: txpool, - genesis: gspec, - } -} - -func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } -func (b *testWorkerBackend) TxPool() *txpool.TxPool { return b.txPool } - -func (b *testWorkerBackend) newRandomTx(creation bool) *types.Transaction { - var tx *types.Transaction - gasPrice := big.NewInt(10 * params.InitialBaseFee) - if creation { - tx, _ = types.SignTx(types.NewContractCreation(b.txPool.Nonce(testBankAddress), big.NewInt(0), testGas, gasPrice, common.FromHex(testCode)), types.HomesteadSigner{}, testBankKey) - } else { - tx, _ = types.SignTx(types.NewTransaction(b.txPool.Nonce(testBankAddress), testUserAddress, big.NewInt(1000), params.TxGas, gasPrice, nil), types.HomesteadSigner{}, testBankKey) - } - return tx -} - -func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*worker, *testWorkerBackend) { - backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) - backend.txPool.Add(pendingTxs, true, false) - w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false) - w.setEtherbase(testBankAddress) - return w, backend -} - -func TestGenerateAndImportBlock(t *testing.T) { - t.Parallel() - var ( - db = rawdb.NewMemoryDatabase() - config = *params.AllCliqueProtocolChanges - ) - config.Clique = ¶ms.CliqueConfig{Period: 1, Epoch: 30000} - engine := clique.New(config.Clique, db) - - w, b := newTestWorker(t, &config, engine, db, 0) - defer w.close() - - // This test chain imports the mined blocks. - chain, _ := core.NewBlockChain(rawdb.NewMemoryDatabase(), nil, b.genesis, nil, engine, vm.Config{}, nil, nil) - defer chain.Stop() - - // Ignore empty commit here for less noise. - w.skipSealHook = func(task *task) bool { - return len(task.receipts) == 0 - } - - // Wait for mined blocks. - sub := w.mux.Subscribe(core.NewMinedBlockEvent{}) - defer sub.Unsubscribe() - - // Start mining! - w.start() - - for i := 0; i < 5; i++ { - b.txPool.Add([]*types.Transaction{b.newRandomTx(true)}, true, false) - b.txPool.Add([]*types.Transaction{b.newRandomTx(false)}, true, false) - - select { - case ev := <-sub.Chan(): - block := ev.Data.(core.NewMinedBlockEvent).Block - if _, err := chain.InsertChain([]*types.Block{block}); err != nil { - t.Fatalf("failed to insert new mined block %d: %v", block.NumberU64(), err) - } - case <-time.After(3 * time.Second): // Worker needs 1s to include new changes. - t.Fatalf("timeout") - } - } -} - -func TestEmptyWorkEthash(t *testing.T) { - t.Parallel() - testEmptyWork(t, ethashChainConfig, ethash.NewFaker()) -} -func TestEmptyWorkClique(t *testing.T) { - t.Parallel() - testEmptyWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) -} - -func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { - defer engine.Close() - - w, _ := newTestWorker(t, chainConfig, engine, rawdb.NewMemoryDatabase(), 0) - defer w.close() - - taskCh := make(chan struct{}, 2) - checkEqual := func(t *testing.T, task *task) { - // The work should contain 1 tx - receiptLen, balance := 1, uint256.NewInt(1000) - if len(task.receipts) != receiptLen { - t.Fatalf("receipt number mismatch: have %d, want %d", len(task.receipts), receiptLen) - } - if task.state.GetBalance(testUserAddress).Cmp(balance) != 0 { - t.Fatalf("account balance mismatch: have %d, want %d", task.state.GetBalance(testUserAddress), balance) - } - } - w.newTaskHook = func(task *task) { - if task.block.NumberU64() == 1 { - checkEqual(t, task) - taskCh <- struct{}{} - } - } - w.skipSealHook = func(task *task) bool { return true } - w.fullTaskHook = func() { - time.Sleep(100 * time.Millisecond) - } - w.start() // Start mining! - select { - case <-taskCh: - case <-time.NewTimer(3 * time.Second).C: - t.Error("new task timeout") - } -} - -func TestAdjustIntervalEthash(t *testing.T) { - t.Parallel() - testAdjustInterval(t, ethashChainConfig, ethash.NewFaker()) -} - -func TestAdjustIntervalClique(t *testing.T) { - t.Parallel() - testAdjustInterval(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) -} - -func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { - defer engine.Close() - - w, _ := newTestWorker(t, chainConfig, engine, rawdb.NewMemoryDatabase(), 0) - defer w.close() - - w.skipSealHook = func(task *task) bool { - return true - } - w.fullTaskHook = func() { - time.Sleep(100 * time.Millisecond) - } - var ( - progress = make(chan struct{}, 10) - result = make([]float64, 0, 10) - index = 0 - start atomic.Bool - ) - w.resubmitHook = func(minInterval time.Duration, recommitInterval time.Duration) { - // Short circuit if interval checking hasn't started. - if !start.Load() { - return - } - var wantMinInterval, wantRecommitInterval time.Duration - - switch index { - case 0: - wantMinInterval, wantRecommitInterval = 3*time.Second, 3*time.Second - case 1: - origin := float64(3 * time.Second.Nanoseconds()) - estimate := origin*(1-intervalAdjustRatio) + intervalAdjustRatio*(origin/0.8+intervalAdjustBias) - wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(estimate)*time.Nanosecond - case 2: - estimate := result[index-1] - min := float64(3 * time.Second.Nanoseconds()) - estimate = estimate*(1-intervalAdjustRatio) + intervalAdjustRatio*(min-intervalAdjustBias) - wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(estimate)*time.Nanosecond - case 3: - wantMinInterval, wantRecommitInterval = time.Second, time.Second - } - - // Check interval - if minInterval != wantMinInterval { - t.Errorf("resubmit min interval mismatch: have %v, want %v ", minInterval, wantMinInterval) - } - if recommitInterval != wantRecommitInterval { - t.Errorf("resubmit interval mismatch: have %v, want %v", recommitInterval, wantRecommitInterval) - } - result = append(result, float64(recommitInterval.Nanoseconds())) - index += 1 - progress <- struct{}{} - } - w.start() - - time.Sleep(time.Second) // Ensure two tasks have been submitted due to start opt - start.Store(true) - - w.setRecommitInterval(3 * time.Second) - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } - - w.resubmitAdjustCh <- &intervalAdjust{inc: true, ratio: 0.8} - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } - - w.resubmitAdjustCh <- &intervalAdjust{inc: false} - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } - - w.setRecommitInterval(500 * time.Millisecond) - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } -} - -func TestGetSealingWorkEthash(t *testing.T) { - t.Parallel() - testGetSealingWork(t, ethashChainConfig, ethash.NewFaker()) -} - -func TestGetSealingWorkClique(t *testing.T) { - t.Parallel() - testGetSealingWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) -} - -func TestGetSealingWorkPostMerge(t *testing.T) { - t.Parallel() - local := new(params.ChainConfig) - *local = *ethashChainConfig - local.TerminalTotalDifficulty = big.NewInt(0) - testGetSealingWork(t, local, ethash.NewFaker()) -} - -func testGetSealingWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { - defer engine.Close() - - w, b := newTestWorker(t, chainConfig, engine, rawdb.NewMemoryDatabase(), 0) - defer w.close() - - w.setExtra([]byte{0x01, 0x02}) - - w.skipSealHook = func(task *task) bool { - return true - } - w.fullTaskHook = func() { - time.Sleep(100 * time.Millisecond) - } - timestamp := uint64(time.Now().Unix()) - assertBlock := func(block *types.Block, number uint64, coinbase common.Address, random common.Hash) { - if block.Time() != timestamp { - // Sometime the timestamp will be mutated if the timestamp - // is even smaller than parent block's. It's OK. - t.Logf("Invalid timestamp, want %d, get %d", timestamp, block.Time()) - } - _, isClique := engine.(*clique.Clique) - if !isClique { - if len(block.Extra()) != 2 { - t.Error("Unexpected extra field") - } - if block.Coinbase() != coinbase { - t.Errorf("Unexpected coinbase got %x want %x", block.Coinbase(), coinbase) - } - } else { - if block.Coinbase() != (common.Address{}) { - t.Error("Unexpected coinbase") - } - } - if !isClique { - if block.MixDigest() != random { - t.Error("Unexpected mix digest") - } - } - if block.Nonce() != 0 { - t.Error("Unexpected block nonce") - } - if block.NumberU64() != number { - t.Errorf("Mismatched block number, want %d got %d", number, block.NumberU64()) - } - } - var cases = []struct { - parent common.Hash - coinbase common.Address - random common.Hash - expectNumber uint64 - expectErr bool - }{ - { - b.chain.Genesis().Hash(), - common.HexToAddress("0xdeadbeef"), - common.HexToHash("0xcafebabe"), - uint64(1), - false, - }, - { - b.chain.CurrentBlock().Hash(), - common.HexToAddress("0xdeadbeef"), - common.HexToHash("0xcafebabe"), - b.chain.CurrentBlock().Number.Uint64() + 1, - false, - }, - { - b.chain.CurrentBlock().Hash(), - common.Address{}, - common.HexToHash("0xcafebabe"), - b.chain.CurrentBlock().Number.Uint64() + 1, - false, - }, - { - b.chain.CurrentBlock().Hash(), - common.Address{}, - common.Hash{}, - b.chain.CurrentBlock().Number.Uint64() + 1, - false, - }, - { - common.HexToHash("0xdeadbeef"), - common.HexToAddress("0xdeadbeef"), - common.HexToHash("0xcafebabe"), - 0, - true, - }, - } - - // This API should work even when the automatic sealing is not enabled - for _, c := range cases { - r := w.getSealingBlock(&generateParams{ - parentHash: c.parent, - timestamp: timestamp, - coinbase: c.coinbase, - random: c.random, - withdrawals: nil, - beaconRoot: nil, - noTxs: false, - forceTime: true, - }) - if c.expectErr { - if r.err == nil { - t.Error("Expect error but get nil") - } - } else { - if r.err != nil { - t.Errorf("Unexpected error %v", r.err) - } - assertBlock(r.block, c.expectNumber, c.coinbase, c.random) - } - } - - // This API should work even when the automatic sealing is enabled - w.start() - for _, c := range cases { - r := w.getSealingBlock(&generateParams{ - parentHash: c.parent, - timestamp: timestamp, - coinbase: c.coinbase, - random: c.random, - withdrawals: nil, - beaconRoot: nil, - noTxs: false, - forceTime: true, - }) - if c.expectErr { - if r.err == nil { - t.Error("Expect error but get nil") - } - } else { - if r.err != nil { - t.Errorf("Unexpected error %v", r.err) - } - assertBlock(r.block, c.expectNumber, c.coinbase, c.random) - } - } -} From aadcb886753079d419f966a3bc990f708f8d1c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Wed, 6 Mar 2024 17:50:22 +0100 Subject: [PATCH 68/91] cmd/blsync, beacon/light: beacon chain light client (#28822) Here we add a beacon chain light client for use by geth. Geth can now be configured to run against a beacon chain API endpoint, without pointing a CL to it. To set this up, use the `--beacon.api` flag. Information provided by the beacon chain is verified, i.e. geth does not blindly trust the beacon API endpoint in this mode. The root of trust are the beacon chain 'sync committees'. The configured beacon API endpoint must provide light client data. At this time, only Lodestar and Nimbus provide the necessary APIs. There is also a standalone tool, cmd/blsync, which uses the beacon chain light client to drive any EL implementation via its engine API. --------- Co-authored-by: Felix Lange --- beacon/blsync/block_sync.go | 203 ++++++++++ beacon/blsync/block_sync_test.go | 160 ++++++++ beacon/blsync/client.go | 103 +++++ beacon/blsync/config.go | 113 ++++++ beacon/light/api/api_server.go | 103 +++++ beacon/light/api/light_api.go | 496 +++++++++++++++++++++++++ beacon/light/committee_chain.go | 25 +- beacon/light/committee_chain_test.go | 4 +- beacon/light/head_tracker.go | 150 ++++++++ beacon/light/request/scheduler.go | 401 ++++++++++++++++++++ beacon/light/request/scheduler_test.go | 122 ++++++ beacon/light/request/server.go | 439 ++++++++++++++++++++++ beacon/light/request/server_test.go | 158 ++++++++ beacon/light/sync/head_sync.go | 176 +++++++++ beacon/light/sync/head_sync_test.go | 151 ++++++++ beacon/light/sync/test_helpers.go | 254 +++++++++++++ beacon/light/sync/types.go | 42 +++ beacon/light/sync/update_sync.go | 299 +++++++++++++++ beacon/light/sync/update_sync_test.go | 219 +++++++++++ beacon/params/params.go | 2 + beacon/types/light_sync.go | 56 +++ cmd/blsync/engine_api.go | 69 ++++ cmd/blsync/main.go | 125 +++++++ cmd/geth/config.go | 3 + cmd/geth/main.go | 8 + cmd/utils/flags.go | 53 +++ eth/catalyst/blsync.go | 88 +++++ go.mod | 5 +- go.sum | 16 +- internal/flags/categories.go | 1 + node/node.go | 24 +- 31 files changed, 4049 insertions(+), 19 deletions(-) create mode 100755 beacon/blsync/block_sync.go create mode 100644 beacon/blsync/block_sync_test.go create mode 100644 beacon/blsync/client.go create mode 100644 beacon/blsync/config.go create mode 100755 beacon/light/api/api_server.go create mode 100755 beacon/light/api/light_api.go create mode 100644 beacon/light/head_tracker.go create mode 100644 beacon/light/request/scheduler.go create mode 100644 beacon/light/request/scheduler_test.go create mode 100644 beacon/light/request/server.go create mode 100644 beacon/light/request/server_test.go create mode 100644 beacon/light/sync/head_sync.go create mode 100644 beacon/light/sync/head_sync_test.go create mode 100644 beacon/light/sync/test_helpers.go create mode 100644 beacon/light/sync/types.go create mode 100644 beacon/light/sync/update_sync.go create mode 100644 beacon/light/sync/update_sync_test.go create mode 100644 cmd/blsync/engine_api.go create mode 100644 cmd/blsync/main.go create mode 100644 eth/catalyst/blsync.go diff --git a/beacon/blsync/block_sync.go b/beacon/blsync/block_sync.go new file mode 100755 index 000000000000..91b21163e655 --- /dev/null +++ b/beacon/blsync/block_sync.go @@ -0,0 +1,203 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + ctypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/zrnt/eth2/configs" + "github.com/protolambda/ztyp/tree" +) + +// beaconBlockSync implements request.Module; it fetches the beacon blocks belonging +// to the validated and prefetch heads. +type beaconBlockSync struct { + recentBlocks *lru.Cache[common.Hash, *capella.BeaconBlock] + locked map[common.Hash]request.ServerAndID + serverHeads map[request.Server]common.Hash + headTracker headTracker + + lastHeadInfo types.HeadInfo + chainHeadFeed *event.Feed +} + +type headTracker interface { + PrefetchHead() types.HeadInfo + ValidatedHead() (types.SignedHeader, bool) + ValidatedFinality() (types.FinalityUpdate, bool) +} + +// newBeaconBlockSync returns a new beaconBlockSync. +func newBeaconBlockSync(headTracker headTracker, chainHeadFeed *event.Feed) *beaconBlockSync { + return &beaconBlockSync{ + headTracker: headTracker, + chainHeadFeed: chainHeadFeed, + recentBlocks: lru.NewCache[common.Hash, *capella.BeaconBlock](10), + locked: make(map[common.Hash]request.ServerAndID), + serverHeads: make(map[request.Server]common.Hash), + } +} + +// Process implements request.Module. +func (s *beaconBlockSync) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + switch event.Type { + case request.EvResponse, request.EvFail, request.EvTimeout: + sid, req, resp := event.RequestInfo() + blockRoot := common.Hash(req.(sync.ReqBeaconBlock)) + if resp != nil { + s.recentBlocks.Add(blockRoot, resp.(*capella.BeaconBlock)) + } + if s.locked[blockRoot] == sid { + delete(s.locked, blockRoot) + } + case sync.EvNewHead: + s.serverHeads[event.Server] = event.Data.(types.HeadInfo).BlockRoot + case request.EvUnregistered: + delete(s.serverHeads, event.Server) + } + } + s.updateEventFeed() + // request validated head block if unavailable and not yet requested + if vh, ok := s.headTracker.ValidatedHead(); ok { + s.tryRequestBlock(requester, vh.Header.Hash(), false) + } + // request prefetch head if the given server has announced it + if prefetchHead := s.headTracker.PrefetchHead().BlockRoot; prefetchHead != (common.Hash{}) { + s.tryRequestBlock(requester, prefetchHead, true) + } +} + +func (s *beaconBlockSync) tryRequestBlock(requester request.Requester, blockRoot common.Hash, needSameHead bool) { + if _, ok := s.recentBlocks.Get(blockRoot); ok { + return + } + if _, ok := s.locked[blockRoot]; ok { + return + } + for _, server := range requester.CanSendTo() { + if needSameHead && (s.serverHeads[server] != blockRoot) { + continue + } + id := requester.Send(server, sync.ReqBeaconBlock(blockRoot)) + s.locked[blockRoot] = request.ServerAndID{Server: server, ID: id} + return + } +} + +func blockHeadInfo(block *capella.BeaconBlock) types.HeadInfo { + if block == nil { + return types.HeadInfo{} + } + return types.HeadInfo{Slot: uint64(block.Slot), BlockRoot: beaconBlockHash(block)} +} + +// beaconBlockHash calculates the hash of a beacon block. +func beaconBlockHash(beaconBlock *capella.BeaconBlock) common.Hash { + return common.Hash(beaconBlock.HashTreeRoot(configs.Mainnet, tree.GetHashFn())) +} + +// getExecBlock extracts the execution block from the beacon block's payload. +func getExecBlock(beaconBlock *capella.BeaconBlock) (*ctypes.Block, error) { + payload := &beaconBlock.Body.ExecutionPayload + txs := make([]*ctypes.Transaction, len(payload.Transactions)) + for i, opaqueTx := range payload.Transactions { + var tx ctypes.Transaction + if err := tx.UnmarshalBinary(opaqueTx); err != nil { + return nil, fmt.Errorf("failed to parse tx %d: %v", i, err) + } + txs[i] = &tx + } + withdrawals := make([]*ctypes.Withdrawal, len(payload.Withdrawals)) + for i, w := range payload.Withdrawals { + withdrawals[i] = &ctypes.Withdrawal{ + Index: uint64(w.Index), + Validator: uint64(w.ValidatorIndex), + Address: common.Address(w.Address), + Amount: uint64(w.Amount), + } + } + wroot := ctypes.DeriveSha(ctypes.Withdrawals(withdrawals), trie.NewStackTrie(nil)) + execHeader := &ctypes.Header{ + ParentHash: common.Hash(payload.ParentHash), + UncleHash: ctypes.EmptyUncleHash, + Coinbase: common.Address(payload.FeeRecipient), + Root: common.Hash(payload.StateRoot), + TxHash: ctypes.DeriveSha(ctypes.Transactions(txs), trie.NewStackTrie(nil)), + ReceiptHash: common.Hash(payload.ReceiptsRoot), + Bloom: ctypes.Bloom(payload.LogsBloom), + Difficulty: common.Big0, + Number: new(big.Int).SetUint64(uint64(payload.BlockNumber)), + GasLimit: uint64(payload.GasLimit), + GasUsed: uint64(payload.GasUsed), + Time: uint64(payload.Timestamp), + Extra: []byte(payload.ExtraData), + MixDigest: common.Hash(payload.PrevRandao), // reused in merge + Nonce: ctypes.BlockNonce{}, // zero + BaseFee: (*uint256.Int)(&payload.BaseFeePerGas).ToBig(), + WithdrawalsHash: &wroot, + } + execBlock := ctypes.NewBlockWithHeader(execHeader).WithBody(txs, nil).WithWithdrawals(withdrawals) + if execBlockHash := execBlock.Hash(); execBlockHash != common.Hash(payload.BlockHash) { + return execBlock, fmt.Errorf("Sanity check failed, payload hash does not match (expected %x, got %x)", common.Hash(payload.BlockHash), execBlockHash) + } + return execBlock, nil +} + +func (s *beaconBlockSync) updateEventFeed() { + head, ok := s.headTracker.ValidatedHead() + if !ok { + return + } + finality, ok := s.headTracker.ValidatedFinality() //TODO fetch directly if subscription does not deliver + if !ok || head.Header.Epoch() != finality.Attested.Header.Epoch() { + return + } + validatedHead := head.Header.Hash() + headBlock, ok := s.recentBlocks.Get(validatedHead) + if !ok { + return + } + headInfo := blockHeadInfo(headBlock) + if headInfo == s.lastHeadInfo { + return + } + s.lastHeadInfo = headInfo + // new head block and finality info available; extract executable data and send event to feed + execBlock, err := getExecBlock(headBlock) + if err != nil { + log.Error("Error extracting execution block from validated beacon block", "error", err) + return + } + s.chainHeadFeed.Send(types.ChainHeadEvent{ + HeadBlock: engine.BlockToExecutableData(execBlock, nil, nil).ExecutionPayload, + Finalized: common.Hash(finality.Finalized.PayloadHeader.BlockHash), + }) +} diff --git a/beacon/blsync/block_sync_test.go b/beacon/blsync/block_sync_test.go new file mode 100644 index 000000000000..9ce434d86273 --- /dev/null +++ b/beacon/blsync/block_sync_test.go @@ -0,0 +1,160 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "testing" + + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/event" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/zrnt/eth2/configs" + "github.com/protolambda/ztyp/tree" +) + +var ( + testServer1 = "testServer1" + testServer2 = "testServer2" + + testBlock1 = &capella.BeaconBlock{ + Slot: 123, + Body: capella.BeaconBlockBody{ + ExecutionPayload: capella.ExecutionPayload{BlockNumber: 456}, + }, + } + testBlock2 = &capella.BeaconBlock{ + Slot: 124, + Body: capella.BeaconBlockBody{ + ExecutionPayload: capella.ExecutionPayload{BlockNumber: 457}, + }, + } +) + +func init() { + eb1, _ := getExecBlock(testBlock1) + testBlock1.Body.ExecutionPayload.BlockHash = tree.Root(eb1.Hash()) + eb2, _ := getExecBlock(testBlock2) + testBlock2.Body.ExecutionPayload.BlockHash = tree.Root(eb2.Hash()) +} + +func TestBlockSync(t *testing.T) { + ht := &testHeadTracker{} + eventFeed := new(event.Feed) + blockSync := newBeaconBlockSync(ht, eventFeed) + headCh := make(chan types.ChainHeadEvent, 16) + eventFeed.Subscribe(headCh) + ts := sync.NewTestScheduler(t, blockSync) + ts.AddServer(testServer1, 1) + ts.AddServer(testServer2, 1) + + expHeadBlock := func(tci int, expHead *capella.BeaconBlock) { + var expNumber, headNumber uint64 + if expHead != nil { + expNumber = uint64(expHead.Body.ExecutionPayload.BlockNumber) + } + select { + case event := <-headCh: + headNumber = event.HeadBlock.Number + default: + } + if headNumber != expNumber { + t.Errorf("Wrong head block in test case #%d (expected block number %d, got %d)", tci, expNumber, headNumber) + } + } + + // no block requests expected until head tracker knows about a head + ts.Run(1) + expHeadBlock(1, nil) + + // set block 1 as prefetch head, announced by server 2 + head1 := blockHeadInfo(testBlock1) + ht.prefetch = head1 + ts.ServerEvent(sync.EvNewHead, testServer2, head1) + // expect request to server 2 which has announced the head + ts.Run(2, testServer2, sync.ReqBeaconBlock(head1.BlockRoot)) + + // valid response + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testBlock1) + ts.AddAllowance(testServer2, 1) + ts.Run(3) + // head block still not expected as the fetched block is not the validated head yet + expHeadBlock(3, nil) + + // set as validated head, expect no further requests but block 1 set as head block + ht.validated.Header = blockHeader(testBlock1) + ts.Run(4) + expHeadBlock(4, testBlock1) + + // set block 2 as prefetch head, announced by server 1 + head2 := blockHeadInfo(testBlock2) + ht.prefetch = head2 + ts.ServerEvent(sync.EvNewHead, testServer1, head2) + // expect request to server 1 + ts.Run(5, testServer1, sync.ReqBeaconBlock(head2.BlockRoot)) + + // req2 fails, no further requests expected because server 2 has not announced it + ts.RequestEvent(request.EvFail, ts.Request(5, 1), nil) + ts.Run(6) + + // set as validated head before retrieving block; now it's assumed to be available from server 2 too + ht.validated.Header = blockHeader(testBlock2) + // expect req2 retry to server 2 + ts.Run(7, testServer2, sync.ReqBeaconBlock(head2.BlockRoot)) + // now head block should be unavailable again + expHeadBlock(4, nil) + + // valid response, now head block should be block 2 immediately as it is already validated + ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testBlock2) + ts.Run(8) + expHeadBlock(5, testBlock2) +} + +func blockHeader(block *capella.BeaconBlock) types.Header { + return types.Header{ + Slot: uint64(block.Slot), + ProposerIndex: uint64(block.ProposerIndex), + ParentRoot: common.Hash(block.ParentRoot), + StateRoot: common.Hash(block.StateRoot), + BodyRoot: common.Hash(block.Body.HashTreeRoot(configs.Mainnet, tree.GetHashFn())), + } +} + +type testHeadTracker struct { + prefetch types.HeadInfo + validated types.SignedHeader +} + +func (h *testHeadTracker) PrefetchHead() types.HeadInfo { + return h.prefetch +} + +func (h *testHeadTracker) ValidatedHead() (types.SignedHeader, bool) { + return h.validated, h.validated.Header != (types.Header{}) +} + +// TODO add test case for finality +func (h *testHeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) { + return types.FinalityUpdate{ + Attested: types.HeaderWithExecProof{Header: h.validated.Header}, + Finalized: types.HeaderWithExecProof{PayloadHeader: &capella.ExecutionPayloadHeader{}}, + Signature: h.validated.Signature, + SignatureSlot: h.validated.SignatureSlot, + }, h.validated.Header != (types.Header{}) +} diff --git a/beacon/blsync/client.go b/beacon/blsync/client.go new file mode 100644 index 000000000000..1bfbb1316069 --- /dev/null +++ b/beacon/blsync/client.go @@ -0,0 +1,103 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "strings" + + "github.com/ethereum/go-ethereum/beacon/light" + "github.com/ethereum/go-ethereum/beacon/light/api" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/event" + "github.com/urfave/cli/v2" +) + +type Client struct { + scheduler *request.Scheduler + chainHeadFeed *event.Feed + urls []string + customHeader map[string]string +} + +func NewClient(ctx *cli.Context) *Client { + if !ctx.IsSet(utils.BeaconApiFlag.Name) { + utils.Fatalf("Beacon node light client API URL not specified") + } + var ( + chainConfig = makeChainConfig(ctx) + customHeader = make(map[string]string) + ) + for _, s := range ctx.StringSlice(utils.BeaconApiHeaderFlag.Name) { + kv := strings.Split(s, ":") + if len(kv) != 2 { + utils.Fatalf("Invalid custom API header entry: %s", s) + } + customHeader[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) + } + // create data structures + var ( + db = memorydb.New() + threshold = ctx.Int(utils.BeaconThresholdFlag.Name) + committeeChain = light.NewCommitteeChain(db, chainConfig.ChainConfig, threshold, !ctx.Bool(utils.BeaconNoFilterFlag.Name)) + headTracker = light.NewHeadTracker(committeeChain, threshold) + ) + headSync := sync.NewHeadSync(headTracker, committeeChain) + + // set up scheduler and sync modules + chainHeadFeed := new(event.Feed) + scheduler := request.NewScheduler() + checkpointInit := sync.NewCheckpointInit(committeeChain, chainConfig.Checkpoint) + forwardSync := sync.NewForwardUpdateSync(committeeChain) + beaconBlockSync := newBeaconBlockSync(headTracker, chainHeadFeed) + scheduler.RegisterTarget(headTracker) + scheduler.RegisterTarget(committeeChain) + scheduler.RegisterModule(checkpointInit, "checkpointInit") + scheduler.RegisterModule(forwardSync, "forwardSync") + scheduler.RegisterModule(headSync, "headSync") + scheduler.RegisterModule(beaconBlockSync, "beaconBlockSync") + + return &Client{ + scheduler: scheduler, + urls: ctx.StringSlice(utils.BeaconApiFlag.Name), + customHeader: customHeader, + chainHeadFeed: chainHeadFeed, + } +} + +// SubscribeChainHeadEvent allows callers to subscribe a provided channel to new +// head updates. +func (c *Client) SubscribeChainHeadEvent(ch chan<- types.ChainHeadEvent) event.Subscription { + return c.chainHeadFeed.Subscribe(ch) +} + +func (c *Client) Start() { + c.scheduler.Start() + // register server(s) + for _, url := range c.urls { + beaconApi := api.NewBeaconLightApi(url, c.customHeader) + c.scheduler.RegisterServer(request.NewServer(api.NewApiServer(beaconApi), &mclock.System{})) + } +} + +func (c *Client) Stop() { + c.scheduler.Stop() +} diff --git a/beacon/blsync/config.go b/beacon/blsync/config.go new file mode 100644 index 000000000000..b51d3e2b0566 --- /dev/null +++ b/beacon/blsync/config.go @@ -0,0 +1,113 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/urfave/cli/v2" +) + +// lightClientConfig contains beacon light client configuration +type lightClientConfig struct { + *types.ChainConfig + Checkpoint common.Hash +} + +var ( + MainnetConfig = lightClientConfig{ + ChainConfig: (&types.ChainConfig{ + GenesisValidatorsRoot: common.HexToHash("0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"), + GenesisTime: 1606824023, + }). + AddFork("GENESIS", 0, []byte{0, 0, 0, 0}). + AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}). + AddFork("BELLATRIX", 144896, []byte{2, 0, 0, 0}). + AddFork("CAPELLA", 194048, []byte{3, 0, 0, 0}), + Checkpoint: common.HexToHash("0x388be41594ec7d6a6894f18c73f3469f07e2c19a803de4755d335817ed8e2e5a"), + } + + SepoliaConfig = lightClientConfig{ + ChainConfig: (&types.ChainConfig{ + GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"), + GenesisTime: 1655733600, + }). + AddFork("GENESIS", 0, []byte{144, 0, 0, 105}). + AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}). + AddFork("BELLATRIX", 100, []byte{144, 0, 0, 113}). + AddFork("CAPELLA", 56832, []byte{144, 0, 0, 114}), + Checkpoint: common.HexToHash("0x1005a6d9175e96bfbce4d35b80f468e9bff0b674e1e861d16e09e10005a58e81"), + } + + GoerliConfig = lightClientConfig{ + ChainConfig: (&types.ChainConfig{ + GenesisValidatorsRoot: common.HexToHash("0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb"), + GenesisTime: 1614588812, + }). + AddFork("GENESIS", 0, []byte{0, 0, 16, 32}). + AddFork("ALTAIR", 36660, []byte{1, 0, 16, 32}). + AddFork("BELLATRIX", 112260, []byte{2, 0, 16, 32}). + AddFork("CAPELLA", 162304, []byte{3, 0, 16, 32}), + Checkpoint: common.HexToHash("0x53a0f4f0a378e2c4ae0a9ee97407eb69d0d737d8d8cd0a5fb1093f42f7b81c49"), + } +) + +func makeChainConfig(ctx *cli.Context) lightClientConfig { + utils.CheckExclusive(ctx, utils.MainnetFlag, utils.GoerliFlag, utils.SepoliaFlag) + customConfig := ctx.IsSet(utils.BeaconConfigFlag.Name) || ctx.IsSet(utils.BeaconGenesisRootFlag.Name) || ctx.IsSet(utils.BeaconGenesisTimeFlag.Name) + var config lightClientConfig + switch { + case ctx.Bool(utils.MainnetFlag.Name): + config = MainnetConfig + case ctx.Bool(utils.SepoliaFlag.Name): + config = SepoliaConfig + case ctx.Bool(utils.GoerliFlag.Name): + config = GoerliConfig + default: + if !customConfig { + config = MainnetConfig + } + } + if customConfig && config.Forks != nil { + utils.Fatalf("Cannot use custom beacon chain config flags in combination with pre-defined network config") + } + if ctx.IsSet(utils.BeaconGenesisRootFlag.Name) { + if c, err := hexutil.Decode(ctx.String(utils.BeaconGenesisRootFlag.Name)); err == nil && len(c) <= 32 { + copy(config.GenesisValidatorsRoot[:len(c)], c) + } else { + utils.Fatalf("Invalid hex string", "beacon.genesis.gvroot", ctx.String(utils.BeaconGenesisRootFlag.Name), "error", err) + } + } + if ctx.IsSet(utils.BeaconGenesisTimeFlag.Name) { + config.GenesisTime = ctx.Uint64(utils.BeaconGenesisTimeFlag.Name) + } + if ctx.IsSet(utils.BeaconConfigFlag.Name) { + if err := config.ChainConfig.LoadForks(ctx.String(utils.BeaconConfigFlag.Name)); err != nil { + utils.Fatalf("Could not load beacon chain config file", "file name", ctx.String(utils.BeaconConfigFlag.Name), "error", err) + } + } + if ctx.IsSet(utils.BeaconCheckpointFlag.Name) { + if c, err := hexutil.Decode(ctx.String(utils.BeaconCheckpointFlag.Name)); err == nil && len(c) <= 32 { + copy(config.Checkpoint[:len(c)], c) + } else { + utils.Fatalf("Invalid hex string", "beacon.checkpoint", ctx.String(utils.BeaconCheckpointFlag.Name), "error", err) + } + } + return config +} diff --git a/beacon/light/api/api_server.go b/beacon/light/api/api_server.go new file mode 100755 index 000000000000..da044f4b2d6e --- /dev/null +++ b/beacon/light/api/api_server.go @@ -0,0 +1,103 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "reflect" + + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +// ApiServer is a wrapper around BeaconLightApi that implements request.requestServer. +type ApiServer struct { + api *BeaconLightApi + eventCallback func(event request.Event) + unsubscribe func() +} + +// NewApiServer creates a new ApiServer. +func NewApiServer(api *BeaconLightApi) *ApiServer { + return &ApiServer{api: api} +} + +// Subscribe implements request.requestServer. +func (s *ApiServer) Subscribe(eventCallback func(event request.Event)) { + s.eventCallback = eventCallback + listener := HeadEventListener{ + OnNewHead: func(slot uint64, blockRoot common.Hash) { + log.Debug("New head received", "slot", slot, "blockRoot", blockRoot) + eventCallback(request.Event{Type: sync.EvNewHead, Data: types.HeadInfo{Slot: slot, BlockRoot: blockRoot}}) + }, + OnSignedHead: func(head types.SignedHeader) { + log.Debug("New signed head received", "slot", head.Header.Slot, "blockRoot", head.Header.Hash(), "signerCount", head.Signature.SignerCount()) + eventCallback(request.Event{Type: sync.EvNewSignedHead, Data: head}) + }, + OnFinality: func(head types.FinalityUpdate) { + log.Debug("New finality update received", "slot", head.Attested.Slot, "blockRoot", head.Attested.Hash(), "signerCount", head.Signature.SignerCount()) + eventCallback(request.Event{Type: sync.EvNewFinalityUpdate, Data: head}) + }, + OnError: func(err error) { + log.Warn("Head event stream error", "err", err) + }, + } + s.unsubscribe = s.api.StartHeadListener(listener) +} + +// SendRequest implements request.requestServer. +func (s *ApiServer) SendRequest(id request.ID, req request.Request) { + go func() { + var resp request.Response + var err error + switch data := req.(type) { + case sync.ReqUpdates: + log.Debug("Beacon API: requesting light client update", "reqid", id, "period", data.FirstPeriod, "count", data.Count) + var r sync.RespUpdates + r.Updates, r.Committees, err = s.api.GetBestUpdatesAndCommittees(data.FirstPeriod, data.Count) + resp = r + case sync.ReqHeader: + log.Debug("Beacon API: requesting header", "reqid", id, "hash", common.Hash(data)) + resp, err = s.api.GetHeader(common.Hash(data)) + case sync.ReqCheckpointData: + log.Debug("Beacon API: requesting checkpoint data", "reqid", id, "hash", common.Hash(data)) + resp, err = s.api.GetCheckpointData(common.Hash(data)) + case sync.ReqBeaconBlock: + log.Debug("Beacon API: requesting block", "reqid", id, "hash", common.Hash(data)) + resp, err = s.api.GetBeaconBlock(common.Hash(data)) + default: + } + + if err != nil { + log.Warn("Beacon API request failed", "type", reflect.TypeOf(req), "reqid", id, "err", err) + s.eventCallback(request.Event{Type: request.EvFail, Data: request.RequestResponse{ID: id, Request: req}}) + } else { + s.eventCallback(request.Event{Type: request.EvResponse, Data: request.RequestResponse{ID: id, Request: req, Response: resp}}) + } + }() +} + +// Unsubscribe implements request.requestServer. +// Note: Unsubscribe should not be called concurrently with Subscribe. +func (s *ApiServer) Unsubscribe() { + if s.unsubscribe != nil { + s.unsubscribe() + s.unsubscribe = nil + } +} diff --git a/beacon/light/api/light_api.go b/beacon/light/api/light_api.go new file mode 100755 index 000000000000..fd701dc0a871 --- /dev/null +++ b/beacon/light/api/light_api.go @@ -0,0 +1,496 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more detaiapi. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" + + "github.com/donovanhide/eventsource" + "github.com/ethereum/go-ethereum/beacon/merkle" + "github.com/ethereum/go-ethereum/beacon/params" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/zrnt/eth2/configs" + "github.com/protolambda/ztyp/tree" +) + +var ( + ErrNotFound = errors.New("404 Not Found") + ErrInternal = errors.New("500 Internal Server Error") +) + +type CommitteeUpdate struct { + Version string + Update types.LightClientUpdate + NextSyncCommittee types.SerializedSyncCommittee +} + +// See data structure definition here: +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientupdate +type committeeUpdateJson struct { + Version string `json:"version"` + Data committeeUpdateData `json:"data"` +} + +type committeeUpdateData struct { + Header jsonBeaconHeader `json:"attested_header"` + NextSyncCommittee types.SerializedSyncCommittee `json:"next_sync_committee"` + NextSyncCommitteeBranch merkle.Values `json:"next_sync_committee_branch"` + FinalizedHeader *jsonBeaconHeader `json:"finalized_header,omitempty"` + FinalityBranch merkle.Values `json:"finality_branch,omitempty"` + SyncAggregate types.SyncAggregate `json:"sync_aggregate"` + SignatureSlot common.Decimal `json:"signature_slot"` +} + +type jsonBeaconHeader struct { + Beacon types.Header `json:"beacon"` +} + +type jsonHeaderWithExecProof struct { + Beacon types.Header `json:"beacon"` + Execution *capella.ExecutionPayloadHeader `json:"execution"` + ExecutionBranch merkle.Values `json:"execution_branch"` +} + +// UnmarshalJSON unmarshals from JSON. +func (u *CommitteeUpdate) UnmarshalJSON(input []byte) error { + var dec committeeUpdateJson + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + u.Version = dec.Version + u.NextSyncCommittee = dec.Data.NextSyncCommittee + u.Update = types.LightClientUpdate{ + AttestedHeader: types.SignedHeader{ + Header: dec.Data.Header.Beacon, + Signature: dec.Data.SyncAggregate, + SignatureSlot: uint64(dec.Data.SignatureSlot), + }, + NextSyncCommitteeRoot: u.NextSyncCommittee.Root(), + NextSyncCommitteeBranch: dec.Data.NextSyncCommitteeBranch, + FinalityBranch: dec.Data.FinalityBranch, + } + if dec.Data.FinalizedHeader != nil { + u.Update.FinalizedHeader = &dec.Data.FinalizedHeader.Beacon + } + return nil +} + +// fetcher is an interface useful for debug-harnessing the http api. +type fetcher interface { + Do(req *http.Request) (*http.Response, error) +} + +// BeaconLightApi requests light client information from a beacon node REST API. +// Note: all required API endpoints are currently only implemented by Lodestar. +type BeaconLightApi struct { + url string + client fetcher + customHeaders map[string]string +} + +func NewBeaconLightApi(url string, customHeaders map[string]string) *BeaconLightApi { + return &BeaconLightApi{ + url: url, + client: &http.Client{ + Timeout: time.Second * 10, + }, + customHeaders: customHeaders, + } +} + +func (api *BeaconLightApi) httpGet(path string) ([]byte, error) { + req, err := http.NewRequest("GET", api.url+path, nil) + if err != nil { + return nil, err + } + for k, v := range api.customHeaders { + req.Header.Set(k, v) + } + resp, err := api.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + switch resp.StatusCode { + case 200: + return io.ReadAll(resp.Body) + case 404: + return nil, ErrNotFound + case 500: + return nil, ErrInternal + default: + return nil, fmt.Errorf("unexpected error from API endpoint \"%s\": status code %d", path, resp.StatusCode) + } +} + +func (api *BeaconLightApi) httpGetf(format string, params ...any) ([]byte, error) { + return api.httpGet(fmt.Sprintf(format, params...)) +} + +// GetBestUpdateAndCommittee fetches and validates LightClientUpdate for given +// period and full serialized committee for the next period (committee root hash +// equals update.NextSyncCommitteeRoot). +// Note that the results are validated but the update signature should be verified +// by the caller as its validity depends on the update chain. +func (api *BeaconLightApi) GetBestUpdatesAndCommittees(firstPeriod, count uint64) ([]*types.LightClientUpdate, []*types.SerializedSyncCommittee, error) { + resp, err := api.httpGetf("/eth/v1/beacon/light_client/updates?start_period=%d&count=%d", firstPeriod, count) + if err != nil { + return nil, nil, err + } + + var data []CommitteeUpdate + if err := json.Unmarshal(resp, &data); err != nil { + return nil, nil, err + } + if len(data) != int(count) { + return nil, nil, errors.New("invalid number of committee updates") + } + updates := make([]*types.LightClientUpdate, int(count)) + committees := make([]*types.SerializedSyncCommittee, int(count)) + for i, d := range data { + if d.Update.AttestedHeader.Header.SyncPeriod() != firstPeriod+uint64(i) { + return nil, nil, errors.New("wrong committee update header period") + } + if err := d.Update.Validate(); err != nil { + return nil, nil, err + } + if d.NextSyncCommittee.Root() != d.Update.NextSyncCommitteeRoot { + return nil, nil, errors.New("wrong sync committee root") + } + updates[i], committees[i] = new(types.LightClientUpdate), new(types.SerializedSyncCommittee) + *updates[i], *committees[i] = d.Update, d.NextSyncCommittee + } + return updates, committees, nil +} + +// GetOptimisticHeadUpdate fetches a signed header based on the latest available +// optimistic update. Note that the signature should be verified by the caller +// as its validity depends on the update chain. +// +// See data structure definition here: +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientoptimisticupdate +func (api *BeaconLightApi) GetOptimisticHeadUpdate() (types.SignedHeader, error) { + resp, err := api.httpGet("/eth/v1/beacon/light_client/optimistic_update") + if err != nil { + return types.SignedHeader{}, err + } + return decodeOptimisticHeadUpdate(resp) +} + +func decodeOptimisticHeadUpdate(enc []byte) (types.SignedHeader, error) { + var data struct { + Data struct { + Header jsonBeaconHeader `json:"attested_header"` + Aggregate types.SyncAggregate `json:"sync_aggregate"` + SignatureSlot common.Decimal `json:"signature_slot"` + } `json:"data"` + } + if err := json.Unmarshal(enc, &data); err != nil { + return types.SignedHeader{}, err + } + if data.Data.Header.Beacon.StateRoot == (common.Hash{}) { + // workaround for different event encoding format in Lodestar + if err := json.Unmarshal(enc, &data.Data); err != nil { + return types.SignedHeader{}, err + } + } + + if len(data.Data.Aggregate.Signers) != params.SyncCommitteeBitmaskSize { + return types.SignedHeader{}, errors.New("invalid sync_committee_bits length") + } + if len(data.Data.Aggregate.Signature) != params.BLSSignatureSize { + return types.SignedHeader{}, errors.New("invalid sync_committee_signature length") + } + return types.SignedHeader{ + Header: data.Data.Header.Beacon, + Signature: data.Data.Aggregate, + SignatureSlot: uint64(data.Data.SignatureSlot), + }, nil +} + +// GetFinalityUpdate fetches the latest available finality update. +// +// See data structure definition here: +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientfinalityupdate +func (api *BeaconLightApi) GetFinalityUpdate() (types.FinalityUpdate, error) { + resp, err := api.httpGet("/eth/v1/beacon/light_client/finality_update") + if err != nil { + return types.FinalityUpdate{}, err + } + return decodeFinalityUpdate(resp) +} + +func decodeFinalityUpdate(enc []byte) (types.FinalityUpdate, error) { + var data struct { + Data struct { + Attested jsonHeaderWithExecProof `json:"attested_header"` + Finalized jsonHeaderWithExecProof `json:"finalized_header"` + FinalityBranch merkle.Values `json:"finality_branch"` + Aggregate types.SyncAggregate `json:"sync_aggregate"` + SignatureSlot common.Decimal `json:"signature_slot"` + } `json:"data"` + } + if err := json.Unmarshal(enc, &data); err != nil { + return types.FinalityUpdate{}, err + } + + if len(data.Data.Aggregate.Signers) != params.SyncCommitteeBitmaskSize { + return types.FinalityUpdate{}, errors.New("invalid sync_committee_bits length") + } + if len(data.Data.Aggregate.Signature) != params.BLSSignatureSize { + return types.FinalityUpdate{}, errors.New("invalid sync_committee_signature length") + } + return types.FinalityUpdate{ + Attested: types.HeaderWithExecProof{ + Header: data.Data.Attested.Beacon, + PayloadHeader: data.Data.Attested.Execution, + PayloadBranch: data.Data.Attested.ExecutionBranch, + }, + Finalized: types.HeaderWithExecProof{ + Header: data.Data.Finalized.Beacon, + PayloadHeader: data.Data.Finalized.Execution, + PayloadBranch: data.Data.Finalized.ExecutionBranch, + }, + FinalityBranch: data.Data.FinalityBranch, + Signature: data.Data.Aggregate, + SignatureSlot: uint64(data.Data.SignatureSlot), + }, nil +} + +// GetHead fetches and validates the beacon header with the given blockRoot. +// If blockRoot is null hash then the latest head header is fetched. +func (api *BeaconLightApi) GetHeader(blockRoot common.Hash) (types.Header, error) { + var blockId string + if blockRoot == (common.Hash{}) { + blockId = "head" + } else { + blockId = blockRoot.Hex() + } + resp, err := api.httpGetf("/eth/v1/beacon/headers/%s", blockId) + if err != nil { + return types.Header{}, err + } + + var data struct { + Data struct { + Root common.Hash `json:"root"` + Canonical bool `json:"canonical"` + Header struct { + Message types.Header `json:"message"` + Signature hexutil.Bytes `json:"signature"` + } `json:"header"` + } `json:"data"` + } + if err := json.Unmarshal(resp, &data); err != nil { + return types.Header{}, err + } + header := data.Data.Header.Message + if blockRoot == (common.Hash{}) { + blockRoot = data.Data.Root + } + if header.Hash() != blockRoot { + return types.Header{}, errors.New("retrieved beacon header root does not match") + } + return header, nil +} + +// GetCheckpointData fetches and validates bootstrap data belonging to the given checkpoint. +func (api *BeaconLightApi) GetCheckpointData(checkpointHash common.Hash) (*types.BootstrapData, error) { + resp, err := api.httpGetf("/eth/v1/beacon/light_client/bootstrap/0x%x", checkpointHash[:]) + if err != nil { + return nil, err + } + + // See data structure definition here: + // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientbootstrap + type bootstrapData struct { + Data struct { + Header jsonBeaconHeader `json:"header"` + Committee *types.SerializedSyncCommittee `json:"current_sync_committee"` + CommitteeBranch merkle.Values `json:"current_sync_committee_branch"` + } `json:"data"` + } + + var data bootstrapData + if err := json.Unmarshal(resp, &data); err != nil { + return nil, err + } + if data.Data.Committee == nil { + return nil, errors.New("sync committee is missing") + } + header := data.Data.Header.Beacon + if header.Hash() != checkpointHash { + return nil, fmt.Errorf("invalid checkpoint block header, have %v want %v", header.Hash(), checkpointHash) + } + checkpoint := &types.BootstrapData{ + Header: header, + CommitteeBranch: data.Data.CommitteeBranch, + CommitteeRoot: data.Data.Committee.Root(), + Committee: data.Data.Committee, + } + if err := checkpoint.Validate(); err != nil { + return nil, fmt.Errorf("invalid checkpoint: %w", err) + } + if checkpoint.Header.Hash() != checkpointHash { + return nil, errors.New("wrong checkpoint hash") + } + return checkpoint, nil +} + +func (api *BeaconLightApi) GetBeaconBlock(blockRoot common.Hash) (*capella.BeaconBlock, error) { + resp, err := api.httpGetf("/eth/v2/beacon/blocks/0x%x", blockRoot) + if err != nil { + return nil, err + } + + var beaconBlockMessage struct { + Data struct { + Message capella.BeaconBlock `json:"message"` + } `json:"data"` + } + if err := json.Unmarshal(resp, &beaconBlockMessage); err != nil { + return nil, fmt.Errorf("invalid block json data: %v", err) + } + beaconBlock := new(capella.BeaconBlock) + *beaconBlock = beaconBlockMessage.Data.Message + root := common.Hash(beaconBlock.HashTreeRoot(configs.Mainnet, tree.GetHashFn())) + if root != blockRoot { + return nil, fmt.Errorf("Beacon block root hash mismatch (expected: %x, got: %x)", blockRoot, root) + } + return beaconBlock, nil +} + +func decodeHeadEvent(enc []byte) (uint64, common.Hash, error) { + var data struct { + Slot common.Decimal `json:"slot"` + Block common.Hash `json:"block"` + } + if err := json.Unmarshal(enc, &data); err != nil { + return 0, common.Hash{}, err + } + return uint64(data.Slot), data.Block, nil +} + +type HeadEventListener struct { + OnNewHead func(slot uint64, blockRoot common.Hash) + OnSignedHead func(head types.SignedHeader) + OnFinality func(head types.FinalityUpdate) + OnError func(err error) +} + +// StartHeadListener creates an event subscription for heads and signed (optimistic) +// head updates and calls the specified callback functions when they are received. +// The callbacks are also called for the current head and optimistic head at startup. +// They are never called concurrently. +func (api *BeaconLightApi) StartHeadListener(listener HeadEventListener) func() { + closeCh := make(chan struct{}) // initiate closing the stream + closedCh := make(chan struct{}) // stream closed (or failed to create) + stoppedCh := make(chan struct{}) // sync loop stopped + streamCh := make(chan *eventsource.Stream, 1) + go func() { + defer close(closedCh) + // when connected to a Lodestar node the subscription blocks until the + // first actual event arrives; therefore we create the subscription in + // a separate goroutine while letting the main goroutine sync up to the + // current head + req, err := http.NewRequest("GET", api.url+ + "/eth/v1/events?topics=head&topics=light_client_optimistic_update&topics=light_client_finality_update", nil) + if err != nil { + listener.OnError(fmt.Errorf("error creating event subscription request: %v", err)) + return + } + for k, v := range api.customHeaders { + req.Header.Set(k, v) + } + stream, err := eventsource.SubscribeWithRequest("", req) + if err != nil { + listener.OnError(fmt.Errorf("error creating event subscription: %v", err)) + close(streamCh) + return + } + streamCh <- stream + <-closeCh + stream.Close() + }() + + go func() { + defer close(stoppedCh) + + if head, err := api.GetHeader(common.Hash{}); err == nil { + listener.OnNewHead(head.Slot, head.Hash()) + } + if signedHead, err := api.GetOptimisticHeadUpdate(); err == nil { + listener.OnSignedHead(signedHead) + } + if finalityUpdate, err := api.GetFinalityUpdate(); err == nil { + listener.OnFinality(finalityUpdate) + } + stream := <-streamCh + if stream == nil { + return + } + + for { + select { + case event, ok := <-stream.Events: + if !ok { + break + } + switch event.Event() { + case "head": + if slot, blockRoot, err := decodeHeadEvent([]byte(event.Data())); err == nil { + listener.OnNewHead(slot, blockRoot) + } else { + listener.OnError(fmt.Errorf("error decoding head event: %v", err)) + } + case "light_client_optimistic_update": + if signedHead, err := decodeOptimisticHeadUpdate([]byte(event.Data())); err == nil { + listener.OnSignedHead(signedHead) + } else { + listener.OnError(fmt.Errorf("error decoding optimistic update event: %v", err)) + } + case "light_client_finality_update": + if finalityUpdate, err := decodeFinalityUpdate([]byte(event.Data())); err == nil { + listener.OnFinality(finalityUpdate) + } else { + listener.OnError(fmt.Errorf("error decoding finality update event: %v", err)) + } + default: + listener.OnError(fmt.Errorf("unexpected event: %s", event.Event())) + } + case err, ok := <-stream.Errors: + if !ok { + break + } + listener.OnError(err) + } + } + }() + return func() { + close(closeCh) + <-closedCh + <-stoppedCh + } +} diff --git a/beacon/light/committee_chain.go b/beacon/light/committee_chain.go index d707f8cc34da..a8d032bb65c5 100644 --- a/beacon/light/committee_chain.go +++ b/beacon/light/committee_chain.go @@ -70,6 +70,7 @@ type CommitteeChain struct { committees *canonicalStore[*types.SerializedSyncCommittee] fixedCommitteeRoots *canonicalStore[common.Hash] committeeCache *lru.Cache[uint64, syncCommittee] // cache deserialized committees + changeCounter uint64 clock mclock.Clock // monotonic clock (simulated clock in tests) unixNano func() int64 // system clock (simulated clock in tests) @@ -86,6 +87,11 @@ func NewCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signer return newCommitteeChain(db, config, signerThreshold, enforceTime, blsVerifier{}, &mclock.System{}, func() int64 { return time.Now().UnixNano() }) } +// NewTestCommitteeChain creates a new CommitteeChain for testing. +func NewTestCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, clock *mclock.Simulated) *CommitteeChain { + return newCommitteeChain(db, config, signerThreshold, enforceTime, dummyVerifier{}, clock, func() int64 { return int64(clock.Now()) }) +} + // newCommitteeChain creates a new CommitteeChain with the option of replacing the // clock source and signature verification for testing purposes. func newCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, sigVerifier committeeSigVerifier, clock mclock.Clock, unixNano func() int64) *CommitteeChain { @@ -181,20 +187,20 @@ func (s *CommitteeChain) Reset() { if err := s.rollback(0); err != nil { log.Error("Error writing batch into chain database", "error", err) } + s.changeCounter++ } -// CheckpointInit initializes a CommitteeChain based on the checkpoint. +// CheckpointInit initializes a CommitteeChain based on a checkpoint. // Note: if the chain is already initialized and the committees proven by the // checkpoint do match the existing chain then the chain is retained and the // new checkpoint becomes fixed. -func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error { +func (s *CommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error { s.chainmu.Lock() defer s.chainmu.Unlock() if err := bootstrap.Validate(); err != nil { return err } - period := bootstrap.Header.SyncPeriod() if err := s.deleteFixedCommitteeRootsFrom(period + 2); err != nil { s.Reset() @@ -215,6 +221,7 @@ func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error { s.Reset() return err } + s.changeCounter++ return nil } @@ -367,6 +374,7 @@ func (s *CommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommi return ErrWrongCommitteeRoot } } + s.changeCounter++ if reorg { if err := s.rollback(period + 1); err != nil { return err @@ -405,6 +413,13 @@ func (s *CommitteeChain) NextSyncPeriod() (uint64, bool) { return s.committees.periods.End - 1, true } +func (s *CommitteeChain) ChangeCounter() uint64 { + s.chainmu.RLock() + defer s.chainmu.RUnlock() + + return s.changeCounter +} + // rollback removes all committees and fixed roots from the given period and updates // starting from the previous period. func (s *CommitteeChain) rollback(period uint64) error { @@ -452,12 +467,12 @@ func (s *CommitteeChain) getSyncCommittee(period uint64) (syncCommittee, error) if sc, ok := s.committees.get(s.db, period); ok { c, err := s.sigVerifier.deserializeSyncCommittee(sc) if err != nil { - return nil, fmt.Errorf("Sync committee #%d deserialization error: %v", period, err) + return nil, fmt.Errorf("sync committee #%d deserialization error: %v", period, err) } s.committeeCache.Add(period, c) return c, nil } - return nil, fmt.Errorf("Missing serialized sync committee #%d", period) + return nil, fmt.Errorf("missing serialized sync committee #%d", period) } // VerifySignedHeader returns true if the given signed header has a valid signature diff --git a/beacon/light/committee_chain_test.go b/beacon/light/committee_chain_test.go index 60ea2a0efdbf..57b6d7175cce 100644 --- a/beacon/light/committee_chain_test.go +++ b/beacon/light/committee_chain_test.go @@ -241,12 +241,12 @@ func newCommitteeChainTest(t *testing.T, config types.ChainConfig, signerThresho signerThreshold: signerThreshold, enforceTime: enforceTime, } - c.chain = newCommitteeChain(c.db, &config, signerThreshold, enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) }) + c.chain = NewTestCommitteeChain(c.db, &config, signerThreshold, enforceTime, c.clock) return c } func (c *committeeChainTest) reloadChain() { - c.chain = newCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) }) + c.chain = NewTestCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, c.clock) } func (c *committeeChainTest) setClockPeriod(period float64) { diff --git a/beacon/light/head_tracker.go b/beacon/light/head_tracker.go new file mode 100644 index 000000000000..579e1b53daa9 --- /dev/null +++ b/beacon/light/head_tracker.go @@ -0,0 +1,150 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package light + +import ( + "errors" + "sync" + "time" + + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/log" +) + +// HeadTracker keeps track of the latest validated head and the "prefetch" head +// which is the (not necessarily validated) head announced by the majority of +// servers. +type HeadTracker struct { + lock sync.RWMutex + committeeChain *CommitteeChain + minSignerCount int + signedHead types.SignedHeader + hasSignedHead bool + finalityUpdate types.FinalityUpdate + hasFinalityUpdate bool + prefetchHead types.HeadInfo + changeCounter uint64 +} + +// NewHeadTracker creates a new HeadTracker. +func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int) *HeadTracker { + return &HeadTracker{ + committeeChain: committeeChain, + minSignerCount: minSignerCount, + } +} + +// ValidatedHead returns the latest validated head. +func (h *HeadTracker) ValidatedHead() (types.SignedHeader, bool) { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.signedHead, h.hasSignedHead +} + +// ValidatedHead returns the latest validated head. +func (h *HeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.finalityUpdate, h.hasFinalityUpdate +} + +// Validate validates the given signed head. If the head is successfully validated +// and it is better than the old validated head (higher slot or same slot and more +// signers) then ValidatedHead is updated. The boolean return flag signals if +// ValidatedHead has been changed. +func (h *HeadTracker) ValidateHead(head types.SignedHeader) (bool, error) { + h.lock.Lock() + defer h.lock.Unlock() + + replace, err := h.validate(head, h.signedHead) + if replace { + h.signedHead, h.hasSignedHead = head, true + h.changeCounter++ + } + return replace, err +} + +func (h *HeadTracker) ValidateFinality(update types.FinalityUpdate) (bool, error) { + h.lock.Lock() + defer h.lock.Unlock() + + replace, err := h.validate(update.SignedHeader(), h.finalityUpdate.SignedHeader()) + if replace { + h.finalityUpdate, h.hasFinalityUpdate = update, true + h.changeCounter++ + } + return replace, err +} + +func (h *HeadTracker) validate(head, oldHead types.SignedHeader) (bool, error) { + signerCount := head.Signature.SignerCount() + if signerCount < h.minSignerCount { + return false, errors.New("low signer count") + } + if head.Header.Slot < oldHead.Header.Slot || (head.Header.Slot == oldHead.Header.Slot && signerCount <= oldHead.Signature.SignerCount()) { + return false, nil + } + sigOk, age, err := h.committeeChain.VerifySignedHeader(head) + if err != nil { + return false, err + } + if age < 0 { + log.Warn("Future signed head received", "age", age) + } + if age > time.Minute*2 { + log.Warn("Old signed head received", "age", age) + } + if !sigOk { + return false, errors.New("invalid header signature") + } + return true, nil +} + +// PrefetchHead returns the latest known prefetch head's head info. +// This head can be used to start fetching related data hoping that it will be +// validated soon. +// Note that the prefetch head cannot be validated cryptographically so it should +// only be used as a performance optimization hint. +func (h *HeadTracker) PrefetchHead() types.HeadInfo { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.prefetchHead +} + +// SetPrefetchHead sets the prefetch head info. +// Note that HeadTracker does not verify the prefetch head, just acts as a thread +// safe bulletin board. +func (h *HeadTracker) SetPrefetchHead(head types.HeadInfo) { + h.lock.Lock() + defer h.lock.Unlock() + + if head == h.prefetchHead { + return + } + h.prefetchHead = head + h.changeCounter++ +} + +func (h *HeadTracker) ChangeCounter() uint64 { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.changeCounter +} diff --git a/beacon/light/request/scheduler.go b/beacon/light/request/scheduler.go new file mode 100644 index 000000000000..20f811900ecf --- /dev/null +++ b/beacon/light/request/scheduler.go @@ -0,0 +1,401 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package request + +import ( + "sync" + + "github.com/ethereum/go-ethereum/log" +) + +// Module represents a mechanism which is typically responsible for downloading +// and updating a passive data structure. It does not directly interact with the +// servers. It can start requests using the Requester interface, maintain its +// internal state by receiving and processing Events and update its target data +// structure based on the obtained data. +// It is the Scheduler's responsibility to feed events to the modules, call +// Process as long as there might be something to process and then generate request +// candidates using MakeRequest and start the best possible requests. +// Modules are called by Scheduler whenever a global trigger is fired. All events +// fire the trigger. Changing a target data structure also triggers a next +// processing round as it could make further actions possible either by the same +// or another Module. +type Module interface { + // Process is a non-blocking function responsible for starting requests, + // processing events and updating the target data structures(s) and the + // internal state of the module. Module state typically consists of information + // about pending requests and registered servers. + // Process is always called after an event is received or after a target data + // structure has been changed. + // + // Note: Process functions of different modules are never called concurrently; + // they are called by Scheduler in the same order of priority as they were + // registered in. + Process(Requester, []Event) +} + +// Requester allows Modules to obtain the list of momentarily available servers, +// start new requests and report server failure when a response has been proven +// to be invalid in the processing phase. +// Note that all Requester functions should be safe to call from Module.Process. +type Requester interface { + CanSendTo() []Server + Send(Server, Request) ID + Fail(Server, string) +} + +// Scheduler is a modular network data retrieval framework that coordinates multiple +// servers and retrieval mechanisms (modules). It implements a trigger mechanism +// that calls the Process function of registered modules whenever either the state +// of existing data structures or events coming from registered servers could +// allow new operations. +type Scheduler struct { + lock sync.Mutex + modules []Module // first has highest priority + names map[Module]string + servers map[server]struct{} + targets map[targetData]uint64 + + requesterLock sync.RWMutex + serverOrder []server + pending map[ServerAndID]pendingRequest + + // eventLock guards access to the events list. Note that eventLock can be + // locked either while lock is locked or unlocked but lock cannot be locked + // while eventLock is locked. + eventLock sync.Mutex + events []Event + stopCh chan chan struct{} + + triggerCh chan struct{} // restarts waiting sync loop + // if trigger has already been fired then send to testWaitCh blocks until + // the triggered processing round is finished + testWaitCh chan struct{} +} + +type ( + // Server identifies a server without allowing any direct interaction. + // Note: server interface is used by Scheduler and Tracker but not used by + // the modules that do not interact with them directly. + // In order to make module testing easier, Server interface is used in + // events and modules. + Server any + Request any + Response any + ID uint64 + ServerAndID struct { + Server Server + ID ID + } +) + +// targetData represents a registered target data structure that increases its +// ChangeCounter whenever it has been changed. +type targetData interface { + ChangeCounter() uint64 +} + +// pendingRequest keeps track of sent and not yet finalized requests and their +// sender modules. +type pendingRequest struct { + request Request + module Module +} + +// NewScheduler creates a new Scheduler. +func NewScheduler() *Scheduler { + s := &Scheduler{ + servers: make(map[server]struct{}), + names: make(map[Module]string), + pending: make(map[ServerAndID]pendingRequest), + targets: make(map[targetData]uint64), + stopCh: make(chan chan struct{}), + // Note: testWaitCh should not have capacity in order to ensure + // that after a trigger happens testWaitCh will block until the resulting + // processing round has been finished + triggerCh: make(chan struct{}, 1), + testWaitCh: make(chan struct{}), + } + return s +} + +// RegisterTarget registers a target data structure, ensuring that any changes +// made to it trigger a new round of Module.Process calls, giving a chance to +// modules to react to the changes. +func (s *Scheduler) RegisterTarget(t targetData) { + s.lock.Lock() + defer s.lock.Unlock() + + s.targets[t] = 0 +} + +// RegisterModule registers a module. Should be called before starting the scheduler. +// In each processing round the order of module processing depends on the order of +// registration. +func (s *Scheduler) RegisterModule(m Module, name string) { + s.lock.Lock() + defer s.lock.Unlock() + + s.modules = append(s.modules, m) + s.names[m] = name +} + +// RegisterServer registers a new server. +func (s *Scheduler) RegisterServer(server server) { + s.lock.Lock() + defer s.lock.Unlock() + + s.addEvent(Event{Type: EvRegistered, Server: server}) + server.subscribe(func(event Event) { + event.Server = server + s.addEvent(event) + }) +} + +// UnregisterServer removes a registered server. +func (s *Scheduler) UnregisterServer(server server) { + s.lock.Lock() + defer s.lock.Unlock() + + server.unsubscribe() + s.addEvent(Event{Type: EvUnregistered, Server: server}) +} + +// Start starts the scheduler. It should be called after registering all modules +// and before registering any servers. +func (s *Scheduler) Start() { + go s.syncLoop() +} + +// Stop stops the scheduler. +func (s *Scheduler) Stop() { + stop := make(chan struct{}) + s.stopCh <- stop + <-stop + s.lock.Lock() + for server := range s.servers { + server.unsubscribe() + } + s.servers = nil + s.lock.Unlock() +} + +// syncLoop is the main event loop responsible for event/data processing and +// sending new requests. +// A round of processing starts whenever the global trigger is fired. Triggers +// fired during a processing round ensure that there is going to be a next round. +func (s *Scheduler) syncLoop() { + for { + s.lock.Lock() + s.processRound() + s.lock.Unlock() + loop: + for { + select { + case stop := <-s.stopCh: + close(stop) + return + case <-s.triggerCh: + break loop + case <-s.testWaitCh: + } + } + } +} + +// targetChanged returns true if a registered target data structure has been +// changed since the last call to this function. +func (s *Scheduler) targetChanged() (changed bool) { + for target, counter := range s.targets { + if newCounter := target.ChangeCounter(); newCounter != counter { + s.targets[target] = newCounter + changed = true + } + } + return +} + +// processRound runs an entire processing round. It calls the Process functions +// of all modules, passing all relevant events and repeating Process calls as +// long as any changes have been made to the registered target data structures. +// Once all events have been processed and a stable state has been achieved, +// requests are generated and sent if necessary and possible. +func (s *Scheduler) processRound() { + for { + log.Trace("Processing modules") + filteredEvents := s.filterEvents() + for _, module := range s.modules { + log.Trace("Processing module", "name", s.names[module], "events", len(filteredEvents[module])) + module.Process(requester{s, module}, filteredEvents[module]) + } + if !s.targetChanged() { + break + } + } +} + +// Trigger starts a new processing round. If fired during processing, it ensures +// another full round of processing all modules. +func (s *Scheduler) Trigger() { + select { + case s.triggerCh <- struct{}{}: + default: + } +} + +// addEvent adds an event to be processed in the next round. Note that it can be +// called regardless of the state of the lock mutex, making it safe for use in +// the server event callback. +func (s *Scheduler) addEvent(event Event) { + s.eventLock.Lock() + s.events = append(s.events, event) + s.eventLock.Unlock() + s.Trigger() +} + +// filterEvent sorts each Event either as a request event or a server event, +// depending on its type. Request events are also sorted in a map based on the +// module that originally initiated the request. It also ensures that no events +// related to a server are returned before EvRegistered or after EvUnregistered. +// In case of an EvUnregistered server event it also closes all pending requests +// to the given server by adding a failed request event (EvFail), ensuring that +// all requests get finalized and thereby allowing the module logic to be safe +// and simple. +func (s *Scheduler) filterEvents() map[Module][]Event { + s.eventLock.Lock() + events := s.events + s.events = nil + s.eventLock.Unlock() + + s.requesterLock.Lock() + defer s.requesterLock.Unlock() + + filteredEvents := make(map[Module][]Event) + for _, event := range events { + server := event.Server.(server) + if _, ok := s.servers[server]; !ok && event.Type != EvRegistered { + continue // before EvRegister or after EvUnregister, discard + } + + if event.IsRequestEvent() { + sid, _, _ := event.RequestInfo() + pending, ok := s.pending[sid] + if !ok { + continue // request already closed, ignore further events + } + if event.Type == EvResponse || event.Type == EvFail { + delete(s.pending, sid) // final event, close pending request + } + filteredEvents[pending.module] = append(filteredEvents[pending.module], event) + } else { + switch event.Type { + case EvRegistered: + s.servers[server] = struct{}{} + s.serverOrder = append(s.serverOrder, nil) + copy(s.serverOrder[1:], s.serverOrder[:len(s.serverOrder)-1]) + s.serverOrder[0] = server + case EvUnregistered: + s.closePending(event.Server, filteredEvents) + delete(s.servers, server) + for i, srv := range s.serverOrder { + if srv == server { + copy(s.serverOrder[i:len(s.serverOrder)-1], s.serverOrder[i+1:]) + s.serverOrder = s.serverOrder[:len(s.serverOrder)-1] + break + } + } + } + for _, module := range s.modules { + filteredEvents[module] = append(filteredEvents[module], event) + } + } + } + return filteredEvents +} + +// closePending closes all pending requests to the given server and adds an EvFail +// event to properly finalize them +func (s *Scheduler) closePending(server Server, filteredEvents map[Module][]Event) { + for sid, pending := range s.pending { + if sid.Server == server { + filteredEvents[pending.module] = append(filteredEvents[pending.module], Event{ + Type: EvFail, + Server: server, + Data: RequestResponse{ + ID: sid.ID, + Request: pending.request, + }, + }) + delete(s.pending, sid) + } + } +} + +// requester implements Requester. Note that while requester basically wraps +// Scheduler (with the added information of the currently processed Module), all +// functions are safe to call from Module.Process which is running while +// the Scheduler.lock mutex is held. +type requester struct { + *Scheduler + module Module +} + +// CanSendTo returns the list of currently available servers. It also returns +// them in an order of least to most recently used, ensuring a round-robin usage +// of suitable servers if the module always chooses the first suitable one. +func (s requester) CanSendTo() []Server { + s.requesterLock.RLock() + defer s.requesterLock.RUnlock() + + list := make([]Server, 0, len(s.serverOrder)) + for _, server := range s.serverOrder { + if server.canRequestNow() { + list = append(list, server) + } + } + return list +} + +// Send sends a request and adds an entry to Scheduler.pending map, ensuring that +// related request events will be delivered to the sender Module. +func (s requester) Send(srv Server, req Request) ID { + s.requesterLock.Lock() + defer s.requesterLock.Unlock() + + server := srv.(server) + id := server.sendRequest(req) + sid := ServerAndID{Server: srv, ID: id} + s.pending[sid] = pendingRequest{request: req, module: s.module} + for i, ss := range s.serverOrder { + if ss == server { + copy(s.serverOrder[i:len(s.serverOrder)-1], s.serverOrder[i+1:]) + s.serverOrder[len(s.serverOrder)-1] = server + return id + } + } + log.Error("Target server not found in ordered list of registered servers") + return id +} + +// Fail should be called when a server delivers invalid or useless information. +// Calling Fail disables the given server for a period that is initially short +// but is exponentially growing if it happens frequently. This results in a +// somewhat fault tolerant operation that avoids hammering servers with requests +// that they cannot serve but still gives them a chance periodically. +func (s requester) Fail(srv Server, desc string) { + srv.(server).fail(desc) +} diff --git a/beacon/light/request/scheduler_test.go b/beacon/light/request/scheduler_test.go new file mode 100644 index 000000000000..7d5a56707864 --- /dev/null +++ b/beacon/light/request/scheduler_test.go @@ -0,0 +1,122 @@ +package request + +import ( + "reflect" + "testing" +) + +func TestEventFilter(t *testing.T) { + s := NewScheduler() + module1 := &testModule{name: "module1"} + module2 := &testModule{name: "module2"} + s.RegisterModule(module1, "module1") + s.RegisterModule(module2, "module2") + s.Start() + // startup process round without events + s.testWaitCh <- struct{}{} + module1.expProcess(t, nil) + module2.expProcess(t, nil) + srv := &testServer{} + // register server; both modules should receive server event + s.RegisterServer(srv) + s.testWaitCh <- struct{}{} + module1.expProcess(t, []Event{ + {Type: EvRegistered, Server: srv}, + }) + module2.expProcess(t, []Event{ + {Type: EvRegistered, Server: srv}, + }) + // let module1 send a request + srv.canRequest = 1 + module1.sendReq = testRequest + s.Trigger() + // in first triggered round module1 sends the request, no events yet + s.testWaitCh <- struct{}{} + module1.expProcess(t, nil) + module2.expProcess(t, nil) + // server emits EvTimeout; only module1 should receive it + srv.eventCb(Event{Type: EvTimeout, Data: RequestResponse{ID: 1, Request: testRequest}}) + s.testWaitCh <- struct{}{} + module1.expProcess(t, []Event{ + {Type: EvTimeout, Server: srv, Data: RequestResponse{ID: 1, Request: testRequest}}, + }) + module2.expProcess(t, nil) + // unregister server; both modules should receive server event + s.UnregisterServer(srv) + s.testWaitCh <- struct{}{} + module1.expProcess(t, []Event{ + // module1 should also receive EvFail on its pending request + {Type: EvFail, Server: srv, Data: RequestResponse{ID: 1, Request: testRequest}}, + {Type: EvUnregistered, Server: srv}, + }) + module2.expProcess(t, []Event{ + {Type: EvUnregistered, Server: srv}, + }) + // response after server unregistered; should be discarded + srv.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + s.testWaitCh <- struct{}{} + module1.expProcess(t, nil) + module2.expProcess(t, nil) + // no more process rounds expected; shut down + s.testWaitCh <- struct{}{} + module1.expNoMoreProcess(t) + module2.expNoMoreProcess(t) + s.Stop() +} + +type testServer struct { + eventCb func(Event) + lastID ID + canRequest int +} + +func (s *testServer) subscribe(eventCb func(Event)) { + s.eventCb = eventCb +} + +func (s *testServer) canRequestNow() bool { + return s.canRequest > 0 +} + +func (s *testServer) sendRequest(req Request) ID { + s.canRequest-- + s.lastID++ + return s.lastID +} + +func (s *testServer) fail(string) {} +func (s *testServer) unsubscribe() {} + +type testModule struct { + name string + processed [][]Event + sendReq Request +} + +func (m *testModule) Process(requester Requester, events []Event) { + m.processed = append(m.processed, events) + if m.sendReq != nil { + if cs := requester.CanSendTo(); len(cs) > 0 { + requester.Send(cs[0], m.sendReq) + } + } +} + +func (m *testModule) expProcess(t *testing.T, expEvents []Event) { + if len(m.processed) == 0 { + t.Errorf("Missing call to %s.Process", m.name) + return + } + events := m.processed[0] + m.processed = m.processed[1:] + if !reflect.DeepEqual(events, expEvents) { + t.Errorf("Call to %s.Process with wrong events (expected %v, got %v)", m.name, expEvents, events) + } +} + +func (m *testModule) expNoMoreProcess(t *testing.T) { + for len(m.processed) > 0 { + t.Errorf("Unexpected call to %s.Process with events %v", m.name, m.processed[0]) + m.processed = m.processed[1:] + } +} diff --git a/beacon/light/request/server.go b/beacon/light/request/server.go new file mode 100644 index 000000000000..999f64178af4 --- /dev/null +++ b/beacon/light/request/server.go @@ -0,0 +1,439 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package request + +import ( + "math" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/log" +) + +var ( + // request events + EvResponse = &EventType{Name: "response", requestEvent: true} // data: RequestResponse; sent by requestServer + EvFail = &EventType{Name: "fail", requestEvent: true} // data: RequestResponse; sent by requestServer + EvTimeout = &EventType{Name: "timeout", requestEvent: true} // data: RequestResponse; sent by serverWithTimeout + // server events + EvRegistered = &EventType{Name: "registered"} // data: nil; sent by Scheduler + EvUnregistered = &EventType{Name: "unregistered"} // data: nil; sent by Scheduler + EvCanRequestAgain = &EventType{Name: "canRequestAgain"} // data: nil; sent by serverWithLimits +) + +const ( + softRequestTimeout = time.Second // allow resending request to a different server but do not cancel yet + hardRequestTimeout = time.Second * 10 // cancel request +) + +const ( + // serverWithLimits parameters + parallelAdjustUp = 0.1 // adjust parallelLimit up in case of success under full load + parallelAdjustDown = 1 // adjust parallelLimit down in case of timeout/failure + minParallelLimit = 1 // parallelLimit lower bound + defaultParallelLimit = 3 // parallelLimit initial value + minFailureDelay = time.Millisecond * 100 // minimum disable time in case of request failure + maxFailureDelay = time.Minute // maximum disable time in case of request failure + maxServerEventBuffer = 5 // server event allowance buffer limit + maxServerEventRate = time.Second // server event allowance buffer recharge rate +) + +// requestServer can send requests in a non-blocking way and feed back events +// through the event callback. After each request it should send back either +// EvResponse or EvFail. Additionally, it may also send application-defined +// events that the Modules can interpret. +type requestServer interface { + Subscribe(eventCallback func(Event)) + SendRequest(ID, Request) + Unsubscribe() +} + +// server is implemented by a requestServer wrapped into serverWithTimeout and +// serverWithLimits and is used by Scheduler. +// In addition to requestServer functionality, server can also handle timeouts, +// limit the number of parallel in-flight requests and temporarily disable +// new requests based on timeouts and response failures. +type server interface { + subscribe(eventCallback func(Event)) + canRequestNow() bool + sendRequest(Request) ID + fail(string) + unsubscribe() +} + +// NewServer wraps a requestServer and returns a server +func NewServer(rs requestServer, clock mclock.Clock) server { + s := &serverWithLimits{} + s.parent = rs + s.serverWithTimeout.init(clock) + s.init() + return s +} + +// EventType identifies an event type, either related to a request or the server +// in general. Server events can also be externally defined. +type EventType struct { + Name string + requestEvent bool // all request events are pre-defined in request package +} + +// Event describes an event where the type of Data depends on Type. +// Server field is not required when sent through the event callback; it is filled +// out when processed by the Scheduler. Note that the Scheduler can also create +// and send events (EvRegistered, EvUnregistered) directly. +type Event struct { + Type *EventType + Server Server // filled by Scheduler + Data any +} + +// IsRequestEvent returns true if the event is a request event +func (e *Event) IsRequestEvent() bool { + return e.Type.requestEvent +} + +// RequestInfo assumes that the event is a request event and returns its contents +// in a convenient form. +func (e *Event) RequestInfo() (ServerAndID, Request, Response) { + data := e.Data.(RequestResponse) + return ServerAndID{Server: e.Server, ID: data.ID}, data.Request, data.Response +} + +// RequestResponse is the Data type of request events. +type RequestResponse struct { + ID ID + Request Request + Response Response +} + +// serverWithTimeout wraps a requestServer and introduces timeouts. +// The request's lifecycle is concluded if EvResponse or EvFail emitted by the +// parent requestServer. If this does not happen until softRequestTimeout then +// EvTimeout is emitted, after which the final EvResponse or EvFail is still +// guaranteed to follow. +// If the parent fails to send this final event for hardRequestTimeout then +// serverWithTimeout emits EvFail and discards any further events from the +// parent related to the given request. +type serverWithTimeout struct { + parent requestServer + lock sync.Mutex + clock mclock.Clock + childEventCb func(event Event) + timeouts map[ID]mclock.Timer + lastID ID +} + +// init initializes serverWithTimeout +func (s *serverWithTimeout) init(clock mclock.Clock) { + s.clock = clock + s.timeouts = make(map[ID]mclock.Timer) +} + +// subscribe subscribes to events which include parent (requestServer) events +// plus EvTimeout. +func (s *serverWithTimeout) subscribe(eventCallback func(event Event)) { + s.lock.Lock() + defer s.lock.Unlock() + + s.childEventCb = eventCallback + s.parent.Subscribe(s.eventCallback) +} + +// sendRequest generated a new request ID, emits EvRequest, sets up the timeout +// timer, then sends the request through the parent (requestServer). +func (s *serverWithTimeout) sendRequest(request Request) (reqId ID) { + s.lock.Lock() + s.lastID++ + id := s.lastID + s.startTimeout(RequestResponse{ID: id, Request: request}) + s.lock.Unlock() + s.parent.SendRequest(id, request) + return id +} + +// eventCallback is called by parent (requestServer) event subscription. +func (s *serverWithTimeout) eventCallback(event Event) { + s.lock.Lock() + defer s.lock.Unlock() + + switch event.Type { + case EvResponse, EvFail: + id := event.Data.(RequestResponse).ID + if timer, ok := s.timeouts[id]; ok { + // Note: if stopping the timer is unsuccessful then the resulting AfterFunc + // call will just do nothing + timer.Stop() + delete(s.timeouts, id) + s.childEventCb(event) + } + default: + s.childEventCb(event) + } +} + +// startTimeout starts a timeout timer for the given request. +func (s *serverWithTimeout) startTimeout(reqData RequestResponse) { + id := reqData.ID + s.timeouts[id] = s.clock.AfterFunc(softRequestTimeout, func() { + s.lock.Lock() + if _, ok := s.timeouts[id]; !ok { + s.lock.Unlock() + return + } + s.timeouts[id] = s.clock.AfterFunc(hardRequestTimeout-softRequestTimeout, func() { + s.lock.Lock() + if _, ok := s.timeouts[id]; !ok { + s.lock.Unlock() + return + } + delete(s.timeouts, id) + childEventCb := s.childEventCb + s.lock.Unlock() + childEventCb(Event{Type: EvFail, Data: reqData}) + }) + childEventCb := s.childEventCb + s.lock.Unlock() + childEventCb(Event{Type: EvTimeout, Data: reqData}) + }) +} + +// stop stops all goroutines associated with the server. +func (s *serverWithTimeout) unsubscribe() { + s.lock.Lock() + defer s.lock.Unlock() + + for _, timer := range s.timeouts { + if timer != nil { + timer.Stop() + } + } + s.childEventCb = nil + s.parent.Unsubscribe() +} + +// serverWithLimits wraps serverWithTimeout and implements server. It limits the +// number of parallel in-flight requests and prevents sending new requests when a +// pending one has already timed out. Server events are also rate limited. +// It also implements a failure delay mechanism that adds an exponentially growing +// delay each time a request fails (wrong answer or hard timeout). This makes the +// syncing mechanism less brittle as temporary failures of the server might happen +// sometimes, but still avoids hammering a non-functional server with requests. +type serverWithLimits struct { + serverWithTimeout + lock sync.Mutex + childEventCb func(event Event) + softTimeouts map[ID]struct{} + pendingCount, timeoutCount int + parallelLimit float32 + sendEvent bool + delayTimer mclock.Timer + delayCounter int + failureDelayEnd mclock.AbsTime + failureDelay float64 + serverEventBuffer int + eventBufferUpdated mclock.AbsTime +} + +// init initializes serverWithLimits +func (s *serverWithLimits) init() { + s.softTimeouts = make(map[ID]struct{}) + s.parallelLimit = defaultParallelLimit + s.serverEventBuffer = maxServerEventBuffer +} + +// subscribe subscribes to events which include parent (serverWithTimeout) events +// plus EvCanRequstAgain. +func (s *serverWithLimits) subscribe(eventCallback func(event Event)) { + s.lock.Lock() + defer s.lock.Unlock() + + s.childEventCb = eventCallback + s.serverWithTimeout.subscribe(s.eventCallback) +} + +// eventCallback is called by parent (serverWithTimeout) event subscription. +func (s *serverWithLimits) eventCallback(event Event) { + s.lock.Lock() + var sendCanRequestAgain bool + passEvent := true + switch event.Type { + case EvTimeout: + id := event.Data.(RequestResponse).ID + s.softTimeouts[id] = struct{}{} + s.timeoutCount++ + s.parallelLimit -= parallelAdjustDown + if s.parallelLimit < minParallelLimit { + s.parallelLimit = minParallelLimit + } + log.Debug("Server timeout", "count", s.timeoutCount, "parallelLimit", s.parallelLimit) + case EvResponse, EvFail: + id := event.Data.(RequestResponse).ID + if _, ok := s.softTimeouts[id]; ok { + delete(s.softTimeouts, id) + s.timeoutCount-- + log.Debug("Server timeout finalized", "count", s.timeoutCount, "parallelLimit", s.parallelLimit) + } + if event.Type == EvResponse && s.pendingCount >= int(s.parallelLimit) { + s.parallelLimit += parallelAdjustUp + } + s.pendingCount-- + if s.canRequest() { + sendCanRequestAgain = s.sendEvent + s.sendEvent = false + } + if event.Type == EvFail { + s.failLocked("failed request") + } + default: + // server event; check rate limit + if s.serverEventBuffer < maxServerEventBuffer { + now := s.clock.Now() + sinceUpdate := time.Duration(now - s.eventBufferUpdated) + if sinceUpdate >= maxServerEventRate*time.Duration(maxServerEventBuffer-s.serverEventBuffer) { + s.serverEventBuffer = maxServerEventBuffer + s.eventBufferUpdated = now + } else { + addBuffer := int(sinceUpdate / maxServerEventRate) + s.serverEventBuffer += addBuffer + s.eventBufferUpdated += mclock.AbsTime(maxServerEventRate * time.Duration(addBuffer)) + } + } + if s.serverEventBuffer > 0 { + s.serverEventBuffer-- + } else { + passEvent = false + } + } + childEventCb := s.childEventCb + s.lock.Unlock() + if passEvent { + childEventCb(event) + } + if sendCanRequestAgain { + childEventCb(Event{Type: EvCanRequestAgain}) + } +} + +// sendRequest sends a request through the parent (serverWithTimeout). +func (s *serverWithLimits) sendRequest(request Request) (reqId ID) { + s.lock.Lock() + s.pendingCount++ + s.lock.Unlock() + return s.serverWithTimeout.sendRequest(request) +} + +// stop stops all goroutines associated with the server. +func (s *serverWithLimits) unsubscribe() { + s.lock.Lock() + defer s.lock.Unlock() + + if s.delayTimer != nil { + s.delayTimer.Stop() + s.delayTimer = nil + } + s.childEventCb = nil + s.serverWithTimeout.unsubscribe() +} + +// canRequest checks whether a new request can be started. +func (s *serverWithLimits) canRequest() bool { + if s.delayTimer != nil || s.pendingCount >= int(s.parallelLimit) || s.timeoutCount > 0 { + return false + } + if s.parallelLimit < minParallelLimit { + s.parallelLimit = minParallelLimit + } + return true +} + +// canRequestNow checks whether a new request can be started, according to the +// current in-flight request count and parallelLimit, and also the failure delay +// timer. +// If it returns false then it is guaranteed that an EvCanRequestAgain will be +// sent whenever the server becomes available for requesting again. +func (s *serverWithLimits) canRequestNow() bool { + var sendCanRequestAgain bool + s.lock.Lock() + canRequest := s.canRequest() + if canRequest { + sendCanRequestAgain = s.sendEvent + s.sendEvent = false + } + childEventCb := s.childEventCb + s.lock.Unlock() + if sendCanRequestAgain { + childEventCb(Event{Type: EvCanRequestAgain}) + } + return canRequest +} + +// delay sets the delay timer to the given duration, disabling new requests for +// the given period. +func (s *serverWithLimits) delay(delay time.Duration) { + if s.delayTimer != nil { + // Note: if stopping the timer is unsuccessful then the resulting AfterFunc + // call will just do nothing + s.delayTimer.Stop() + s.delayTimer = nil + } + + s.delayCounter++ + delayCounter := s.delayCounter + log.Debug("Server delay started", "length", delay) + s.delayTimer = s.clock.AfterFunc(delay, func() { + log.Debug("Server delay ended", "length", delay) + var sendCanRequestAgain bool + s.lock.Lock() + if s.delayTimer != nil && s.delayCounter == delayCounter { // do nothing if there is a new timer now + s.delayTimer = nil + if s.canRequest() { + sendCanRequestAgain = s.sendEvent + s.sendEvent = false + } + } + childEventCb := s.childEventCb + s.lock.Unlock() + if sendCanRequestAgain { + childEventCb(Event{Type: EvCanRequestAgain}) + } + }) +} + +// fail reports that a response from the server was found invalid by the processing +// Module, disabling new requests for a dynamically adjused time period. +func (s *serverWithLimits) fail(desc string) { + s.lock.Lock() + defer s.lock.Unlock() + + s.failLocked(desc) +} + +// failLocked calculates the dynamic failure delay and applies it. +func (s *serverWithLimits) failLocked(desc string) { + log.Debug("Server error", "description", desc) + s.failureDelay *= 2 + now := s.clock.Now() + if now > s.failureDelayEnd { + s.failureDelay *= math.Pow(2, -float64(now-s.failureDelayEnd)/float64(maxFailureDelay)) + } + if s.failureDelay < float64(minFailureDelay) { + s.failureDelay = float64(minFailureDelay) + } + s.failureDelayEnd = now + mclock.AbsTime(s.failureDelay) + s.delay(time.Duration(s.failureDelay)) +} diff --git a/beacon/light/request/server_test.go b/beacon/light/request/server_test.go new file mode 100644 index 000000000000..b6b9edf9a056 --- /dev/null +++ b/beacon/light/request/server_test.go @@ -0,0 +1,158 @@ +package request + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +const ( + testRequest = "Life, the Universe, and Everything" + testResponse = 42 +) + +var testEventType = &EventType{Name: "testEvent"} + +func TestServerEvents(t *testing.T) { + rs := &testRequestServer{} + clock := &mclock.Simulated{} + srv := NewServer(rs, clock) + var lastEventType *EventType + srv.subscribe(func(event Event) { lastEventType = event.Type }) + evTypeName := func(evType *EventType) string { + if evType == nil { + return "none" + } + return evType.Name + } + expEvent := func(expType *EventType) { + if lastEventType != expType { + t.Errorf("Wrong event type (expected %s, got %s)", evTypeName(expType), evTypeName(lastEventType)) + } + lastEventType = nil + } + // user events should simply be passed through + rs.eventCb(Event{Type: testEventType}) + expEvent(testEventType) + // send request, soft timeout, then valid response + srv.sendRequest(testRequest) + clock.WaitForTimers(1) + clock.Run(softRequestTimeout) + expEvent(EvTimeout) + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expEvent(EvResponse) + // send request, hard timeout (response after hard timeout should be ignored) + srv.sendRequest(testRequest) + clock.WaitForTimers(1) + clock.Run(softRequestTimeout) + expEvent(EvTimeout) + clock.WaitForTimers(1) + clock.Run(hardRequestTimeout) + expEvent(EvFail) + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expEvent(nil) +} + +func TestServerParallel(t *testing.T) { + rs := &testRequestServer{} + srv := NewServer(rs, &mclock.Simulated{}) + srv.subscribe(func(event Event) {}) + + expSend := func(expSent int) { + var sent int + for sent <= expSent { + if !srv.canRequestNow() { + break + } + sent++ + srv.sendRequest(testRequest) + } + if sent != expSent { + t.Errorf("Wrong number of parallel requests accepted (expected %d, got %d)", expSent, sent) + } + } + // max out parallel allowance + expSend(defaultParallelLimit) + // 1 answered, should accept 1 more + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expSend(1) + // 2 answered, should accept 2 more + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 2, Request: testRequest, Response: testResponse}}) + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 3, Request: testRequest, Response: testResponse}}) + expSend(2) + // failed request, should decrease allowance and not accept more + rs.eventCb(Event{Type: EvFail, Data: RequestResponse{ID: 4, Request: testRequest}}) + expSend(0) + srv.unsubscribe() +} + +func TestServerFail(t *testing.T) { + rs := &testRequestServer{} + clock := &mclock.Simulated{} + srv := NewServer(rs, clock) + srv.subscribe(func(event Event) {}) + expCanRequest := func(expCanRequest bool) { + if canRequest := srv.canRequestNow(); canRequest != expCanRequest { + t.Errorf("Wrong result for canRequestNow (expected %v, got %v)", expCanRequest, canRequest) + } + } + // timed out request + expCanRequest(true) + srv.sendRequest(testRequest) + clock.WaitForTimers(1) + expCanRequest(true) + clock.Run(softRequestTimeout) + expCanRequest(false) // cannot request when there is a timed out request + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expCanRequest(true) + // explicit server.Fail + srv.fail("") + clock.WaitForTimers(1) + expCanRequest(false) // cannot request for a while after a failure + clock.Run(minFailureDelay) + expCanRequest(true) + // request returned with EvFail + srv.sendRequest(testRequest) + rs.eventCb(Event{Type: EvFail, Data: RequestResponse{ID: 2, Request: testRequest}}) + clock.WaitForTimers(1) + expCanRequest(false) // EvFail should also start failure delay + clock.Run(minFailureDelay) + expCanRequest(false) // second failure delay is longer, should still be disabled + clock.Run(minFailureDelay) + expCanRequest(true) + srv.unsubscribe() +} + +func TestServerEventRateLimit(t *testing.T) { + rs := &testRequestServer{} + clock := &mclock.Simulated{} + srv := NewServer(rs, clock) + var eventCount int + srv.subscribe(func(event Event) { + if !event.IsRequestEvent() { + eventCount++ + } + }) + expEvents := func(send, expAllowed int) { + eventCount = 0 + for sent := 0; sent < send; sent++ { + rs.eventCb(Event{Type: testEventType}) + } + if eventCount != expAllowed { + t.Errorf("Wrong number of server events passing rate limitation (sent %d, expected %d, got %d)", send, expAllowed, eventCount) + } + } + expEvents(maxServerEventBuffer+5, maxServerEventBuffer) + clock.Run(maxServerEventRate) + expEvents(5, 1) + clock.Run(maxServerEventRate * maxServerEventBuffer * 2) + expEvents(maxServerEventBuffer+5, maxServerEventBuffer) +} + +type testRequestServer struct { + eventCb func(Event) +} + +func (rs *testRequestServer) Subscribe(eventCb func(Event)) { rs.eventCb = eventCb } +func (rs *testRequestServer) SendRequest(ID, Request) {} +func (rs *testRequestServer) Unsubscribe() {} diff --git a/beacon/light/sync/head_sync.go b/beacon/light/sync/head_sync.go new file mode 100644 index 000000000000..9fef95b0df79 --- /dev/null +++ b/beacon/light/sync/head_sync.go @@ -0,0 +1,176 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" +) + +type headTracker interface { + ValidateHead(head types.SignedHeader) (bool, error) + ValidateFinality(head types.FinalityUpdate) (bool, error) + SetPrefetchHead(head types.HeadInfo) +} + +// HeadSync implements request.Module; it updates the validated and prefetch +// heads of HeadTracker based on the EvHead and EvSignedHead events coming from +// registered servers. +// It can also postpone the validation of the latest announced signed head +// until the committee chain is synced up to at least the required period. +type HeadSync struct { + headTracker headTracker + chain committeeChain + nextSyncPeriod uint64 + chainInit bool + unvalidatedHeads map[request.Server]types.SignedHeader + unvalidatedFinality map[request.Server]types.FinalityUpdate + serverHeads map[request.Server]types.HeadInfo + headServerCount map[types.HeadInfo]headServerCount + headCounter uint64 + prefetchHead types.HeadInfo +} + +// headServerCount is associated with most recently seen head infos; it counts +// the number of servers currently having the given head info as their announced +// head and a counter signaling how recent that head is. +// This data is used for selecting the prefetch head. +type headServerCount struct { + serverCount int + headCounter uint64 +} + +// NewHeadSync creates a new HeadSync. +func NewHeadSync(headTracker headTracker, chain committeeChain) *HeadSync { + s := &HeadSync{ + headTracker: headTracker, + chain: chain, + unvalidatedHeads: make(map[request.Server]types.SignedHeader), + unvalidatedFinality: make(map[request.Server]types.FinalityUpdate), + serverHeads: make(map[request.Server]types.HeadInfo), + headServerCount: make(map[types.HeadInfo]headServerCount), + } + return s +} + +// Process implements request.Module. +func (s *HeadSync) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + switch event.Type { + case EvNewHead: + s.setServerHead(event.Server, event.Data.(types.HeadInfo)) + case EvNewSignedHead: + s.newSignedHead(event.Server, event.Data.(types.SignedHeader)) + case EvNewFinalityUpdate: + s.newFinalityUpdate(event.Server, event.Data.(types.FinalityUpdate)) + case request.EvUnregistered: + s.setServerHead(event.Server, types.HeadInfo{}) + delete(s.serverHeads, event.Server) + delete(s.unvalidatedHeads, event.Server) + } + } + + nextPeriod, chainInit := s.chain.NextSyncPeriod() + if nextPeriod != s.nextSyncPeriod || chainInit != s.chainInit { + s.nextSyncPeriod, s.chainInit = nextPeriod, chainInit + s.processUnvalidated() + } +} + +// newSignedHead handles received signed head; either validates it if the chain +// is properly synced or stores it for further validation. +func (s *HeadSync) newSignedHead(server request.Server, signedHead types.SignedHeader) { + if !s.chainInit || types.SyncPeriod(signedHead.SignatureSlot) > s.nextSyncPeriod { + s.unvalidatedHeads[server] = signedHead + return + } + s.headTracker.ValidateHead(signedHead) +} + +// newSignedHead handles received signed head; either validates it if the chain +// is properly synced or stores it for further validation. +func (s *HeadSync) newFinalityUpdate(server request.Server, finalityUpdate types.FinalityUpdate) { + if !s.chainInit || types.SyncPeriod(finalityUpdate.SignatureSlot) > s.nextSyncPeriod { + s.unvalidatedFinality[server] = finalityUpdate + return + } + s.headTracker.ValidateFinality(finalityUpdate) +} + +// processUnvalidatedHeads iterates the list of unvalidated heads and validates +// those which can be validated. +func (s *HeadSync) processUnvalidated() { + if !s.chainInit { + return + } + for server, signedHead := range s.unvalidatedHeads { + if types.SyncPeriod(signedHead.SignatureSlot) <= s.nextSyncPeriod { + s.headTracker.ValidateHead(signedHead) + delete(s.unvalidatedHeads, server) + } + } + for server, finalityUpdate := range s.unvalidatedFinality { + if types.SyncPeriod(finalityUpdate.SignatureSlot) <= s.nextSyncPeriod { + s.headTracker.ValidateFinality(finalityUpdate) + delete(s.unvalidatedFinality, server) + } + } +} + +// setServerHead processes non-validated server head announcements and updates +// the prefetch head if necessary. +func (s *HeadSync) setServerHead(server request.Server, head types.HeadInfo) bool { + if oldHead, ok := s.serverHeads[server]; ok { + if head == oldHead { + return false + } + h := s.headServerCount[oldHead] + if h.serverCount--; h.serverCount > 0 { + s.headServerCount[oldHead] = h + } else { + delete(s.headServerCount, oldHead) + } + } + if head != (types.HeadInfo{}) { + h, ok := s.headServerCount[head] + if !ok { + s.headCounter++ + h.headCounter = s.headCounter + } + h.serverCount++ + s.headServerCount[head] = h + s.serverHeads[server] = head + } else { + delete(s.serverHeads, server) + } + var ( + bestHead types.HeadInfo + bestHeadInfo headServerCount + ) + for head, headServerCount := range s.headServerCount { + if headServerCount.serverCount > bestHeadInfo.serverCount || + (headServerCount.serverCount == bestHeadInfo.serverCount && headServerCount.headCounter > bestHeadInfo.headCounter) { + bestHead, bestHeadInfo = head, headServerCount + } + } + if bestHead == s.prefetchHead { + return false + } + s.prefetchHead = bestHead + s.headTracker.SetPrefetchHead(bestHead) + return true +} diff --git a/beacon/light/sync/head_sync_test.go b/beacon/light/sync/head_sync_test.go new file mode 100644 index 000000000000..12faad62920e --- /dev/null +++ b/beacon/light/sync/head_sync_test.go @@ -0,0 +1,151 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "testing" + + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" +) + +var ( + testServer1 = "testServer1" + testServer2 = "testServer2" + testServer3 = "testServer3" + testServer4 = "testServer4" + + testHead0 = types.HeadInfo{} + testHead1 = types.HeadInfo{Slot: 123, BlockRoot: common.Hash{1}} + testHead2 = types.HeadInfo{Slot: 124, BlockRoot: common.Hash{2}} + testHead3 = types.HeadInfo{Slot: 124, BlockRoot: common.Hash{3}} + testHead4 = types.HeadInfo{Slot: 125, BlockRoot: common.Hash{4}} + + testSHead1 = types.SignedHeader{SignatureSlot: 0x0124, Header: types.Header{Slot: 0x0123, StateRoot: common.Hash{1}}} + testSHead2 = types.SignedHeader{SignatureSlot: 0x2010, Header: types.Header{Slot: 0x200e, StateRoot: common.Hash{2}}} + // testSHead3 is at the end of period 1 but signed in period 2 + testSHead3 = types.SignedHeader{SignatureSlot: 0x4000, Header: types.Header{Slot: 0x3fff, StateRoot: common.Hash{3}}} + testSHead4 = types.SignedHeader{SignatureSlot: 0x6444, Header: types.Header{Slot: 0x6443, StateRoot: common.Hash{4}}} +) + +func TestValidatedHead(t *testing.T) { + chain := &TestCommitteeChain{} + ht := &TestHeadTracker{} + headSync := NewHeadSync(ht, chain) + ts := NewTestScheduler(t, headSync) + + ht.ExpValidated(t, 0, nil) + + ts.AddServer(testServer1, 1) + ts.ServerEvent(EvNewSignedHead, testServer1, testSHead1) + ts.Run(1) + // announced head should be queued because of uninitialized chain + ht.ExpValidated(t, 1, nil) + + chain.SetNextSyncPeriod(0) // initialize chain + ts.Run(2) + // expect previously queued head to be validated + ht.ExpValidated(t, 2, []types.SignedHeader{testSHead1}) + + chain.SetNextSyncPeriod(1) + ts.ServerEvent(EvNewSignedHead, testServer1, testSHead2) + ts.AddServer(testServer2, 1) + ts.ServerEvent(EvNewSignedHead, testServer2, testSHead2) + ts.Run(3) + // expect both head announcements to be validated instantly + ht.ExpValidated(t, 3, []types.SignedHeader{testSHead2, testSHead2}) + + ts.ServerEvent(EvNewSignedHead, testServer1, testSHead3) + ts.AddServer(testServer3, 1) + ts.ServerEvent(EvNewSignedHead, testServer3, testSHead4) + ts.Run(4) + // future period annonced heads should be queued + ht.ExpValidated(t, 4, nil) + + chain.SetNextSyncPeriod(2) + ts.Run(5) + // testSHead3 can be validated now but not testSHead4 + ht.ExpValidated(t, 5, []types.SignedHeader{testSHead3}) + + // server 3 disconnected without proving period 3, its announced head should be dropped + ts.RemoveServer(testServer3) + ts.Run(6) + ht.ExpValidated(t, 6, nil) + + chain.SetNextSyncPeriod(3) + ts.Run(7) + // testSHead4 could be validated now but it's not queued by any registered server + ht.ExpValidated(t, 7, nil) + + ts.ServerEvent(EvNewSignedHead, testServer2, testSHead4) + ts.Run(8) + // now testSHead4 should be validated + ht.ExpValidated(t, 8, []types.SignedHeader{testSHead4}) +} + +func TestPrefetchHead(t *testing.T) { + chain := &TestCommitteeChain{} + ht := &TestHeadTracker{} + headSync := NewHeadSync(ht, chain) + ts := NewTestScheduler(t, headSync) + + ht.ExpPrefetch(t, 0, testHead0) // no servers registered + + ts.AddServer(testServer1, 1) + ts.ServerEvent(EvNewHead, testServer1, testHead1) + ts.Run(1) + ht.ExpPrefetch(t, 1, testHead1) // s1: h1 + + ts.AddServer(testServer2, 1) + ts.ServerEvent(EvNewHead, testServer2, testHead2) + ts.Run(2) + ht.ExpPrefetch(t, 2, testHead2) // s1: h1, s2: h2 + + ts.ServerEvent(EvNewHead, testServer1, testHead2) + ts.Run(3) + ht.ExpPrefetch(t, 3, testHead2) // s1: h2, s2: h2 + + ts.AddServer(testServer3, 1) + ts.ServerEvent(EvNewHead, testServer3, testHead3) + ts.Run(4) + ht.ExpPrefetch(t, 4, testHead2) // s1: h2, s2: h2, s3: h3 + + ts.AddServer(testServer4, 1) + ts.ServerEvent(EvNewHead, testServer4, testHead4) + ts.Run(5) + ht.ExpPrefetch(t, 5, testHead2) // s1: h2, s2: h2, s3: h3, s4: h4 + + ts.ServerEvent(EvNewHead, testServer2, testHead3) + ts.Run(6) + ht.ExpPrefetch(t, 6, testHead3) // s1: h2, s2: h3, s3: h3, s4: h4 + + ts.RemoveServer(testServer3) + ts.Run(7) + ht.ExpPrefetch(t, 7, testHead4) // s1: h2, s2: h3, s4: h4 + + ts.RemoveServer(testServer1) + ts.Run(8) + ht.ExpPrefetch(t, 8, testHead4) // s2: h3, s4: h4 + + ts.RemoveServer(testServer4) + ts.Run(9) + ht.ExpPrefetch(t, 9, testHead3) // s2: h3 + + ts.RemoveServer(testServer2) + ts.Run(10) + ht.ExpPrefetch(t, 10, testHead0) // no servers registered +} diff --git a/beacon/light/sync/test_helpers.go b/beacon/light/sync/test_helpers.go new file mode 100644 index 000000000000..a1ca2b590993 --- /dev/null +++ b/beacon/light/sync/test_helpers.go @@ -0,0 +1,254 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/beacon/light" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" +) + +type requestWithID struct { + sid request.ServerAndID + request request.Request +} + +type TestScheduler struct { + t *testing.T + module request.Module + events []request.Event + servers []request.Server + allowance map[request.Server]int + sent map[int][]requestWithID + testIndex int + expFail map[request.Server]int // expected Server.Fail calls during next Run + lastId request.ID +} + +func NewTestScheduler(t *testing.T, module request.Module) *TestScheduler { + return &TestScheduler{ + t: t, + module: module, + allowance: make(map[request.Server]int), + expFail: make(map[request.Server]int), + sent: make(map[int][]requestWithID), + } +} + +func (ts *TestScheduler) Run(testIndex int, exp ...any) { + expReqs := make([]requestWithID, len(exp)/2) + id := ts.lastId + for i := range expReqs { + id++ + expReqs[i] = requestWithID{ + sid: request.ServerAndID{Server: exp[i*2].(request.Server), ID: id}, + request: exp[i*2+1].(request.Request), + } + } + if len(expReqs) == 0 { + expReqs = nil + } + + ts.testIndex = testIndex + ts.module.Process(ts, ts.events) + ts.events = nil + + for server, count := range ts.expFail { + delete(ts.expFail, server) + if count == 0 { + continue + } + ts.t.Errorf("Missing %d Server.Fail(s) from server %s in test case #%d", count, server.(string), testIndex) + } + + if !reflect.DeepEqual(ts.sent[testIndex], expReqs) { + ts.t.Errorf("Wrong sent requests in test case #%d (expected %v, got %v)", testIndex, expReqs, ts.sent[testIndex]) + } +} + +func (ts *TestScheduler) CanSendTo() (cs []request.Server) { + for _, server := range ts.servers { + if ts.allowance[server] > 0 { + cs = append(cs, server) + } + } + return +} + +func (ts *TestScheduler) Send(server request.Server, req request.Request) request.ID { + ts.lastId++ + ts.sent[ts.testIndex] = append(ts.sent[ts.testIndex], requestWithID{ + sid: request.ServerAndID{Server: server, ID: ts.lastId}, + request: req, + }) + ts.allowance[server]-- + return ts.lastId +} + +func (ts *TestScheduler) Fail(server request.Server, desc string) { + if ts.expFail[server] == 0 { + ts.t.Errorf("Unexpected Fail from server %s in test case #%d: %s", server.(string), ts.testIndex, desc) + return + } + ts.expFail[server]-- +} + +func (ts *TestScheduler) Request(testIndex, reqIndex int) requestWithID { + if len(ts.sent[testIndex]) < reqIndex { + ts.t.Errorf("Missing request from test case %d index %d", testIndex, reqIndex) + return requestWithID{} + } + return ts.sent[testIndex][reqIndex-1] +} + +func (ts *TestScheduler) ServerEvent(evType *request.EventType, server request.Server, data any) { + ts.events = append(ts.events, request.Event{ + Type: evType, + Server: server, + Data: data, + }) +} + +func (ts *TestScheduler) RequestEvent(evType *request.EventType, req requestWithID, resp request.Response) { + if req.request == nil { + return + } + ts.events = append(ts.events, request.Event{ + Type: evType, + Server: req.sid.Server, + Data: request.RequestResponse{ + ID: req.sid.ID, + Request: req.request, + Response: resp, + }, + }) +} + +func (ts *TestScheduler) AddServer(server request.Server, allowance int) { + ts.servers = append(ts.servers, server) + ts.allowance[server] = allowance + ts.ServerEvent(request.EvRegistered, server, nil) +} + +func (ts *TestScheduler) RemoveServer(server request.Server) { + ts.servers = append(ts.servers, server) + for i, s := range ts.servers { + if s == server { + copy(ts.servers[i:len(ts.servers)-1], ts.servers[i+1:]) + ts.servers = ts.servers[:len(ts.servers)-1] + break + } + } + delete(ts.allowance, server) + ts.ServerEvent(request.EvUnregistered, server, nil) +} + +func (ts *TestScheduler) AddAllowance(server request.Server, allowance int) { + ts.allowance[server] += allowance +} + +func (ts *TestScheduler) ExpFail(server request.Server) { + ts.expFail[server]++ +} + +type TestCommitteeChain struct { + fsp, nsp uint64 + init bool +} + +func (t *TestCommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error { + t.fsp, t.nsp, t.init = bootstrap.Header.SyncPeriod(), bootstrap.Header.SyncPeriod()+2, true + return nil +} + +func (t *TestCommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error { + period := update.AttestedHeader.Header.SyncPeriod() + if period < t.fsp || period > t.nsp || !t.init { + return light.ErrInvalidPeriod + } + if period == t.nsp { + t.nsp++ + } + return nil +} + +func (t *TestCommitteeChain) NextSyncPeriod() (uint64, bool) { + return t.nsp, t.init +} + +func (tc *TestCommitteeChain) ExpInit(t *testing.T, ExpInit bool) { + if tc.init != ExpInit { + t.Errorf("Incorrect init flag (expected %v, got %v)", ExpInit, tc.init) + } +} + +func (t *TestCommitteeChain) SetNextSyncPeriod(nsp uint64) { + t.init, t.nsp = true, nsp +} + +func (tc *TestCommitteeChain) ExpNextSyncPeriod(t *testing.T, expNsp uint64) { + tc.ExpInit(t, true) + if tc.nsp != expNsp { + t.Errorf("Incorrect NextSyncPeriod (expected %d, got %d)", expNsp, tc.nsp) + } +} + +type TestHeadTracker struct { + phead types.HeadInfo + validated []types.SignedHeader +} + +func (ht *TestHeadTracker) ValidateHead(head types.SignedHeader) (bool, error) { + ht.validated = append(ht.validated, head) + return true, nil +} + +// TODO add test case for finality +func (ht *TestHeadTracker) ValidateFinality(head types.FinalityUpdate) (bool, error) { + return true, nil +} + +func (ht *TestHeadTracker) ExpValidated(t *testing.T, tci int, expHeads []types.SignedHeader) { + for i, expHead := range expHeads { + if i >= len(ht.validated) { + t.Errorf("Missing validated head in test case #%d index #%d (expected {slot %d blockRoot %x}, got none)", tci, i, expHead.Header.Slot, expHead.Header.Hash()) + continue + } + if ht.validated[i] != expHead { + vhead := ht.validated[i].Header + t.Errorf("Wrong validated head in test case #%d index #%d (expected {slot %d blockRoot %x}, got {slot %d blockRoot %x})", tci, i, expHead.Header.Slot, expHead.Header.Hash(), vhead.Slot, vhead.Hash()) + } + } + for i := len(expHeads); i < len(ht.validated); i++ { + vhead := ht.validated[i].Header + t.Errorf("Unexpected validated head in test case #%d index #%d (expected none, got {slot %d blockRoot %x})", tci, i, vhead.Slot, vhead.Hash()) + } + ht.validated = nil +} + +func (ht *TestHeadTracker) SetPrefetchHead(head types.HeadInfo) { + ht.phead = head +} + +func (ht *TestHeadTracker) ExpPrefetch(t *testing.T, tci int, exp types.HeadInfo) { + if ht.phead != exp { + t.Errorf("Wrong prefetch head in test case #%d (expected {slot %d blockRoot %x}, got {slot %d blockRoot %x})", tci, exp.Slot, exp.BlockRoot, ht.phead.Slot, ht.phead.BlockRoot) + } +} diff --git a/beacon/light/sync/types.go b/beacon/light/sync/types.go new file mode 100644 index 000000000000..6449ae842d00 --- /dev/null +++ b/beacon/light/sync/types.go @@ -0,0 +1,42 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" +) + +var ( + EvNewHead = &request.EventType{Name: "newHead"} // data: types.HeadInfo + EvNewSignedHead = &request.EventType{Name: "newSignedHead"} // data: types.SignedHeader + EvNewFinalityUpdate = &request.EventType{Name: "newFinalityUpdate"} // data: types.FinalityUpdate +) + +type ( + ReqUpdates struct { + FirstPeriod, Count uint64 + } + RespUpdates struct { + Updates []*types.LightClientUpdate + Committees []*types.SerializedSyncCommittee + } + ReqHeader common.Hash + ReqCheckpointData common.Hash + ReqBeaconBlock common.Hash +) diff --git a/beacon/light/sync/update_sync.go b/beacon/light/sync/update_sync.go new file mode 100644 index 000000000000..533e470fb022 --- /dev/null +++ b/beacon/light/sync/update_sync.go @@ -0,0 +1,299 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "sort" + + "github.com/ethereum/go-ethereum/beacon/light" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +const maxUpdateRequest = 8 // maximum number of updates requested in a single request + +type committeeChain interface { + CheckpointInit(bootstrap types.BootstrapData) error + InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error + NextSyncPeriod() (uint64, bool) +} + +// CheckpointInit implements request.Module; it fetches the light client bootstrap +// data belonging to the given checkpoint hash and initializes the committee chain +// if successful. +type CheckpointInit struct { + chain committeeChain + checkpointHash common.Hash + locked request.ServerAndID + initialized bool +} + +// NewCheckpointInit creates a new CheckpointInit. +func NewCheckpointInit(chain committeeChain, checkpointHash common.Hash) *CheckpointInit { + return &CheckpointInit{ + chain: chain, + checkpointHash: checkpointHash, + } +} + +// Process implements request.Module. +func (s *CheckpointInit) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + if !event.IsRequestEvent() { + continue + } + sid, req, resp := event.RequestInfo() + if s.locked == sid { + s.locked = request.ServerAndID{} + } + if resp != nil { + if checkpoint := resp.(*types.BootstrapData); checkpoint.Header.Hash() == common.Hash(req.(ReqCheckpointData)) { + s.chain.CheckpointInit(*checkpoint) + s.initialized = true + return + } + + requester.Fail(event.Server, "invalid checkpoint data") + } + } + // start a request if possible + if s.initialized || s.locked != (request.ServerAndID{}) { + return + } + cs := requester.CanSendTo() + if len(cs) == 0 { + return + } + server := cs[0] + id := requester.Send(server, ReqCheckpointData(s.checkpointHash)) + s.locked = request.ServerAndID{Server: server, ID: id} +} + +// ForwardUpdateSync implements request.Module; it fetches updates between the +// committee chain head and each server's announced head. Updates are fetched +// in batches and multiple batches can also be requested in parallel. +// Out of order responses are also handled; if a batch of updates cannot be added +// to the chain immediately because of a gap then the future updates are +// remembered until they can be processed. +type ForwardUpdateSync struct { + chain committeeChain + rangeLock rangeLock + lockedIDs map[request.ServerAndID]struct{} + processQueue []updateResponse + nextSyncPeriod map[request.Server]uint64 +} + +// NewForwardUpdateSync creates a new ForwardUpdateSync. +func NewForwardUpdateSync(chain committeeChain) *ForwardUpdateSync { + return &ForwardUpdateSync{ + chain: chain, + rangeLock: make(rangeLock), + lockedIDs: make(map[request.ServerAndID]struct{}), + nextSyncPeriod: make(map[request.Server]uint64), + } +} + +// rangeLock allows locking sections of an integer space, preventing the syncing +// mechanism from making requests again for sections where a not timed out request +// is already pending or where already fetched and unprocessed data is available. +type rangeLock map[uint64]int + +// lock locks or unlocks the given section, depending on the sign of the add parameter. +func (r rangeLock) lock(first, count uint64, add int) { + for i := first; i < first+count; i++ { + if v := r[i] + add; v > 0 { + r[i] = v + } else { + delete(r, i) + } + } +} + +// firstUnlocked returns the first unlocked section starting at or after start +// and not longer than maxCount. +func (r rangeLock) firstUnlocked(start, maxCount uint64) (first, count uint64) { + first = start + for { + if _, ok := r[first]; !ok { + break + } + first++ + } + for { + count++ + if count == maxCount { + break + } + if _, ok := r[first+count]; ok { + break + } + } + return +} + +// lockRange locks the range belonging to the given update request, unless the +// same request has already been locked +func (s *ForwardUpdateSync) lockRange(sid request.ServerAndID, req ReqUpdates) { + if _, ok := s.lockedIDs[sid]; ok { + return + } + s.lockedIDs[sid] = struct{}{} + s.rangeLock.lock(req.FirstPeriod, req.Count, 1) +} + +// unlockRange unlocks the range belonging to the given update request, unless +// same request has already been unlocked +func (s *ForwardUpdateSync) unlockRange(sid request.ServerAndID, req ReqUpdates) { + if _, ok := s.lockedIDs[sid]; !ok { + return + } + delete(s.lockedIDs, sid) + s.rangeLock.lock(req.FirstPeriod, req.Count, -1) +} + +// verifyRange returns true if the number of updates and the individual update +// periods in the response match the requested section. +func (s *ForwardUpdateSync) verifyRange(request ReqUpdates, response RespUpdates) bool { + if uint64(len(response.Updates)) != request.Count || uint64(len(response.Committees)) != request.Count { + return false + } + for i, update := range response.Updates { + if update.AttestedHeader.Header.SyncPeriod() != request.FirstPeriod+uint64(i) { + return false + } + } + return true +} + +// updateResponse is a response that has passed initial verification and has been +// queued for processing. Note that an update response cannot be processed until +// the previous updates have also been added to the chain. +type updateResponse struct { + sid request.ServerAndID + request ReqUpdates + response RespUpdates +} + +// updateResponseList implements sort.Sort and sorts update request/response events by FirstPeriod. +type updateResponseList []updateResponse + +func (u updateResponseList) Len() int { return len(u) } +func (u updateResponseList) Swap(i, j int) { u[i], u[j] = u[j], u[i] } +func (u updateResponseList) Less(i, j int) bool { + return u[i].request.FirstPeriod < u[j].request.FirstPeriod +} + +// Process implements request.Module. +func (s *ForwardUpdateSync) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + switch event.Type { + case request.EvResponse, request.EvFail, request.EvTimeout: + sid, rq, rs := event.RequestInfo() + req := rq.(ReqUpdates) + var queued bool + if event.Type == request.EvResponse { + resp := rs.(RespUpdates) + if s.verifyRange(req, resp) { + // there is a response with a valid format; put it in the process queue + s.processQueue = append(s.processQueue, updateResponse{sid: sid, request: req, response: resp}) + s.lockRange(sid, req) + queued = true + } else { + requester.Fail(event.Server, "invalid update range") + } + } + if !queued { + s.unlockRange(sid, req) + } + case EvNewSignedHead: + signedHead := event.Data.(types.SignedHeader) + s.nextSyncPeriod[event.Server] = types.SyncPeriod(signedHead.SignatureSlot + 256) + case request.EvUnregistered: + delete(s.nextSyncPeriod, event.Server) + } + } + + // try processing ordered list of available responses + sort.Sort(updateResponseList(s.processQueue)) + for s.processQueue != nil { + u := s.processQueue[0] + if !s.processResponse(requester, u) { + break + } + s.unlockRange(u.sid, u.request) + s.processQueue = s.processQueue[1:] + if len(s.processQueue) == 0 { + s.processQueue = nil + } + } + + // start new requests if possible + startPeriod, chainInit := s.chain.NextSyncPeriod() + if !chainInit { + return + } + for { + firstPeriod, maxCount := s.rangeLock.firstUnlocked(startPeriod, maxUpdateRequest) + var ( + sendTo request.Server + bestCount uint64 + ) + for _, server := range requester.CanSendTo() { + nextPeriod := s.nextSyncPeriod[server] + if nextPeriod <= firstPeriod { + continue + } + count := maxCount + if nextPeriod < firstPeriod+maxCount { + count = nextPeriod - firstPeriod + } + if count > bestCount { + sendTo, bestCount = server, count + } + } + if sendTo == nil { + return + } + req := ReqUpdates{FirstPeriod: firstPeriod, Count: bestCount} + id := requester.Send(sendTo, req) + s.lockRange(request.ServerAndID{Server: sendTo, ID: id}, req) + } +} + +// processResponse adds the fetched updates and committees to the committee chain. +// Returns true in case of full or partial success. +func (s *ForwardUpdateSync) processResponse(requester request.Requester, u updateResponse) (success bool) { + for i, update := range u.response.Updates { + if err := s.chain.InsertUpdate(update, u.response.Committees[i]); err != nil { + if err == light.ErrInvalidPeriod { + // there is a gap in the update periods; stop processing without + // failing and try again next time + return + } + if err == light.ErrInvalidUpdate || err == light.ErrWrongCommitteeRoot || err == light.ErrCannotReorg { + requester.Fail(u.sid.Server, "invalid update received") + } else { + log.Error("Unexpected InsertUpdate error", "error", err) + } + return + } + success = true + } + return +} diff --git a/beacon/light/sync/update_sync_test.go b/beacon/light/sync/update_sync_test.go new file mode 100644 index 000000000000..1c4b3d6d76fa --- /dev/null +++ b/beacon/light/sync/update_sync_test.go @@ -0,0 +1,219 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "testing" + + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" +) + +func TestCheckpointInit(t *testing.T) { + chain := &TestCommitteeChain{} + checkpoint := &types.BootstrapData{Header: types.Header{Slot: 0x2000*4 + 0x1000}} // period 4 + checkpointHash := checkpoint.Header.Hash() + chkInit := NewCheckpointInit(chain, checkpointHash) + ts := NewTestScheduler(t, chkInit) + // add 2 servers + ts.AddServer(testServer1, 1) + ts.AddServer(testServer2, 1) + + // expect bootstrap request to server 1 + ts.Run(1, testServer1, ReqCheckpointData(checkpointHash)) + + // server 1 times out; expect request to server 2 + ts.RequestEvent(request.EvTimeout, ts.Request(1, 1), nil) + ts.Run(2, testServer2, ReqCheckpointData(checkpointHash)) + + // invalid response from server 2; expect init state to still be false + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), &types.BootstrapData{Header: types.Header{Slot: 123456}}) + ts.ExpFail(testServer2) + ts.Run(3) + chain.ExpInit(t, false) + + // server 1 fails (hard timeout) + ts.RequestEvent(request.EvFail, ts.Request(1, 1), nil) + ts.Run(4) + chain.ExpInit(t, false) + + // server 3 is registered; expect bootstrap request to server 3 + ts.AddServer(testServer3, 1) + ts.Run(5, testServer3, ReqCheckpointData(checkpointHash)) + + // valid response from server 3; expect chain to be initialized + ts.RequestEvent(request.EvResponse, ts.Request(5, 1), checkpoint) + ts.Run(6) + chain.ExpInit(t, true) +} + +func TestUpdateSyncParallel(t *testing.T) { + chain := &TestCommitteeChain{} + chain.SetNextSyncPeriod(0) + updateSync := NewForwardUpdateSync(chain) + ts := NewTestScheduler(t, updateSync) + // add 2 servers, head at period 100; allow 3-3 parallel requests for each + ts.AddServer(testServer1, 3) + ts.ServerEvent(EvNewSignedHead, testServer1, types.SignedHeader{SignatureSlot: 0x2000*100 + 0x1000}) + ts.AddServer(testServer2, 3) + ts.ServerEvent(EvNewSignedHead, testServer2, types.SignedHeader{SignatureSlot: 0x2000*100 + 0x1000}) + + // expect 6 requests to be sent + ts.Run(1, + testServer1, ReqUpdates{FirstPeriod: 0, Count: 8}, + testServer1, ReqUpdates{FirstPeriod: 8, Count: 8}, + testServer1, ReqUpdates{FirstPeriod: 16, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 24, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 32, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 40, Count: 8}) + + // valid response to request 1; expect 8 periods synced and a new request started + ts.RequestEvent(request.EvResponse, ts.Request(1, 1), testRespUpdate(ts.Request(1, 1))) + ts.AddAllowance(testServer1, 1) + ts.Run(2, testServer1, ReqUpdates{FirstPeriod: 48, Count: 8}) + chain.ExpNextSyncPeriod(t, 8) + + // valid response to requests 4 and 5 + ts.RequestEvent(request.EvResponse, ts.Request(1, 4), testRespUpdate(ts.Request(1, 4))) + ts.RequestEvent(request.EvResponse, ts.Request(1, 5), testRespUpdate(ts.Request(1, 5))) + ts.AddAllowance(testServer2, 2) + // expect 2 more requests but no sync progress (responses 4 and 5 cannot be added before 2 and 3) + ts.Run(3, + testServer2, ReqUpdates{FirstPeriod: 56, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 64, Count: 8}) + chain.ExpNextSyncPeriod(t, 8) + + // soft timeout for requests 2 and 3 (server 1 is overloaded) + ts.RequestEvent(request.EvTimeout, ts.Request(1, 2), nil) + ts.RequestEvent(request.EvTimeout, ts.Request(1, 3), nil) + // no allowance, no more requests + ts.Run(4) + + // valid response to requests 6 and 8 and 9 + ts.RequestEvent(request.EvResponse, ts.Request(1, 6), testRespUpdate(ts.Request(1, 6))) + ts.RequestEvent(request.EvResponse, ts.Request(3, 1), testRespUpdate(ts.Request(3, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(3, 2), testRespUpdate(ts.Request(3, 2))) + ts.AddAllowance(testServer2, 3) + // server 2 can now resend requests 2 and 3 (timed out by server 1) and also send a new one + ts.Run(5, + testServer2, ReqUpdates{FirstPeriod: 8, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 16, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 72, Count: 8}) + + // server 1 finally answers timed out request 2 + ts.RequestEvent(request.EvResponse, ts.Request(1, 2), testRespUpdate(ts.Request(1, 2))) + ts.AddAllowance(testServer1, 1) + // expect sync progress and one new request + ts.Run(6, testServer1, ReqUpdates{FirstPeriod: 80, Count: 8}) + chain.ExpNextSyncPeriod(t, 16) + + // server 2 answers requests 11 and 12 (resends of requests 2 and 3) + ts.RequestEvent(request.EvResponse, ts.Request(5, 1), testRespUpdate(ts.Request(5, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(5, 2), testRespUpdate(ts.Request(5, 2))) + ts.AddAllowance(testServer2, 2) + ts.Run(7, + testServer2, ReqUpdates{FirstPeriod: 88, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 96, Count: 4}) + // finally the gap is filled, update can process responses up to req6 + chain.ExpNextSyncPeriod(t, 48) + + // all remaining requests are answered + ts.RequestEvent(request.EvResponse, ts.Request(1, 3), testRespUpdate(ts.Request(1, 3))) + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testRespUpdate(ts.Request(2, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(5, 3), testRespUpdate(ts.Request(5, 3))) + ts.RequestEvent(request.EvResponse, ts.Request(6, 1), testRespUpdate(ts.Request(6, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testRespUpdate(ts.Request(7, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(7, 2), testRespUpdate(ts.Request(7, 2))) + ts.Run(8) + // expect chain to be fully synced + chain.ExpNextSyncPeriod(t, 100) +} + +func TestUpdateSyncDifferentHeads(t *testing.T) { + chain := &TestCommitteeChain{} + chain.SetNextSyncPeriod(10) + updateSync := NewForwardUpdateSync(chain) + ts := NewTestScheduler(t, updateSync) + // add 3 servers with different announced head periods + ts.AddServer(testServer1, 1) + ts.ServerEvent(EvNewSignedHead, testServer1, types.SignedHeader{SignatureSlot: 0x2000*15 + 0x1000}) + ts.AddServer(testServer2, 1) + ts.ServerEvent(EvNewSignedHead, testServer2, types.SignedHeader{SignatureSlot: 0x2000*16 + 0x1000}) + ts.AddServer(testServer3, 1) + ts.ServerEvent(EvNewSignedHead, testServer3, types.SignedHeader{SignatureSlot: 0x2000*17 + 0x1000}) + + // expect request to the best announced head + ts.Run(1, testServer3, ReqUpdates{FirstPeriod: 10, Count: 7}) + + // request times out, expect request to the next best head + ts.RequestEvent(request.EvTimeout, ts.Request(1, 1), nil) + ts.Run(2, testServer2, ReqUpdates{FirstPeriod: 10, Count: 6}) + + // request times out, expect request to the last available server + ts.RequestEvent(request.EvTimeout, ts.Request(2, 1), nil) + ts.Run(3, testServer1, ReqUpdates{FirstPeriod: 10, Count: 5}) + + // valid response to request 3, expect chain synced to period 15 + ts.RequestEvent(request.EvResponse, ts.Request(3, 1), testRespUpdate(ts.Request(3, 1))) + ts.AddAllowance(testServer1, 1) + ts.Run(4) + chain.ExpNextSyncPeriod(t, 15) + + // invalid response to request 1, server can only deliver updates up to period 15 despite announced head + truncated := ts.Request(1, 1) + truncated.request = ReqUpdates{FirstPeriod: 10, Count: 5} + ts.RequestEvent(request.EvResponse, ts.Request(1, 1), testRespUpdate(truncated)) + ts.ExpFail(testServer3) + ts.Run(5) + // expect no progress of chain head + chain.ExpNextSyncPeriod(t, 15) + + // valid response to request 2, expect chain synced to period 16 + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testRespUpdate(ts.Request(2, 1))) + ts.AddAllowance(testServer2, 1) + ts.Run(6) + chain.ExpNextSyncPeriod(t, 16) + + // a new server is registered with announced head period 17 + ts.AddServer(testServer4, 1) + ts.ServerEvent(EvNewSignedHead, testServer4, types.SignedHeader{SignatureSlot: 0x2000*17 + 0x1000}) + // expect request to sync one more period + ts.Run(7, testServer4, ReqUpdates{FirstPeriod: 16, Count: 1}) + + // valid response, expect chain synced to period 17 + ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testRespUpdate(ts.Request(7, 1))) + ts.AddAllowance(testServer4, 1) + ts.Run(8) + chain.ExpNextSyncPeriod(t, 17) +} + +func testRespUpdate(request requestWithID) request.Response { + var resp RespUpdates + if request.request == nil { + return resp + } + req := request.request.(ReqUpdates) + resp.Updates = make([]*types.LightClientUpdate, int(req.Count)) + resp.Committees = make([]*types.SerializedSyncCommittee, int(req.Count)) + period := req.FirstPeriod + for i := range resp.Updates { + resp.Updates[i] = &types.LightClientUpdate{AttestedHeader: types.SignedHeader{Header: types.Header{Slot: 0x2000*period + 0x1000}}} + resp.Committees[i] = new(types.SerializedSyncCommittee) + period++ + } + return resp +} diff --git a/beacon/params/params.go b/beacon/params/params.go index ee9feb1acbea..e4e0d009340e 100644 --- a/beacon/params/params.go +++ b/beacon/params/params.go @@ -41,4 +41,6 @@ const ( StateIndexNextSyncCommittee = 55 StateIndexExecPayload = 56 StateIndexExecHead = 908 + + BodyIndexExecPayload = 25 ) diff --git a/beacon/types/light_sync.go b/beacon/types/light_sync.go index 3284081e4d49..ed62d237f126 100644 --- a/beacon/types/light_sync.go +++ b/beacon/types/light_sync.go @@ -20,11 +20,20 @@ import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/beacon/merkle" "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/common" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/ztyp/tree" ) +// HeadInfo represents an unvalidated new head announcement. +type HeadInfo struct { + Slot uint64 + BlockRoot common.Hash +} + // BootstrapData contains a sync committee where light sync can be started, // together with a proof through a beacon header and corresponding state. // Note: BootstrapData is fetched from a server based on a known checkpoint hash. @@ -134,3 +143,50 @@ func (u UpdateScore) BetterThan(w UpdateScore) bool { } return u.SignerCount > w.SignerCount } + +type HeaderWithExecProof struct { + Header + PayloadHeader *capella.ExecutionPayloadHeader + PayloadBranch merkle.Values +} + +func (h *HeaderWithExecProof) Validate() error { + payloadRoot := merkle.Value(h.PayloadHeader.HashTreeRoot(tree.GetHashFn())) + return merkle.VerifyProof(h.BodyRoot, params.BodyIndexExecPayload, h.PayloadBranch, payloadRoot) +} + +type FinalityUpdate struct { + Attested, Finalized HeaderWithExecProof + FinalityBranch merkle.Values + // Sync committee BLS signature aggregate + Signature SyncAggregate + // Slot in which the signature has been created (newer than Header.Slot, + // determines the signing sync committee) + SignatureSlot uint64 +} + +func (u *FinalityUpdate) SignedHeader() SignedHeader { + return SignedHeader{ + Header: u.Attested.Header, + Signature: u.Signature, + SignatureSlot: u.SignatureSlot, + } +} + +func (u *FinalityUpdate) Validate() error { + if err := u.Attested.Validate(); err != nil { + return err + } + if err := u.Finalized.Validate(); err != nil { + return err + } + return merkle.VerifyProof(u.Attested.StateRoot, params.StateIndexFinalBlock, u.FinalityBranch, merkle.Value(u.Finalized.Hash())) +} + +// ChainHeadEvent returns an authenticated execution payload associated with the +// latest accepted head of the beacon chain, along with the hash of the latest +// finalized execution block. +type ChainHeadEvent struct { + HeadBlock *engine.ExecutableData + Finalized common.Hash +} diff --git a/cmd/blsync/engine_api.go b/cmd/blsync/engine_api.go new file mode 100644 index 000000000000..d10750e295fe --- /dev/null +++ b/cmd/blsync/engine_api.go @@ -0,0 +1,69 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" +) + +func updateEngineApi(client *rpc.Client, headCh chan types.ChainHeadEvent) { + for event := range headCh { + if client == nil { // dry run, no engine API specified + log.Info("New execution block retrieved", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "finalized block hash", event.Finalized) + } else { + if status, err := callNewPayloadV2(client, event.HeadBlock); err == nil { + log.Info("Successful NewPayload", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "status", status) + } else { + log.Error("Failed NewPayload", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "error", err) + } + if status, err := callForkchoiceUpdatedV1(client, event.HeadBlock.BlockHash, event.Finalized); err == nil { + log.Info("Successful ForkchoiceUpdated", "head", event.HeadBlock.BlockHash, "status", status) + } else { + log.Error("Failed ForkchoiceUpdated", "head", event.HeadBlock.BlockHash, "error", err) + } + } + } +} + +func callNewPayloadV2(client *rpc.Client, execData *engine.ExecutableData) (string, error) { + var resp engine.PayloadStatusV1 + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + err := client.CallContext(ctx, &resp, "engine_newPayloadV2", execData) + cancel() + return resp.Status, err +} + +func callForkchoiceUpdatedV1(client *rpc.Client, headHash, finalizedHash common.Hash) (string, error) { + var resp engine.ForkChoiceResponse + update := engine.ForkchoiceStateV1{ + HeadBlockHash: headHash, + SafeBlockHash: finalizedHash, + FinalizedBlockHash: finalizedHash, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + err := client.CallContext(ctx, &resp, "engine_forkchoiceUpdatedV1", update, nil) + cancel() + return resp.PayloadStatus.Status, err +} diff --git a/cmd/blsync/main.go b/cmd/blsync/main.go new file mode 100644 index 000000000000..fd22761d3c45 --- /dev/null +++ b/cmd/blsync/main.go @@ -0,0 +1,125 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/ethereum/go-ethereum/beacon/blsync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rpc" + "github.com/mattn/go-colorable" + "github.com/mattn/go-isatty" + "github.com/urfave/cli/v2" +) + +var ( + verbosityFlag = &cli.IntFlag{ + Name: "verbosity", + Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail", + Value: 3, + Category: flags.LoggingCategory, + } + vmoduleFlag = &cli.StringFlag{ + Name: "vmodule", + Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", + Value: "", + Hidden: true, + Category: flags.LoggingCategory, + } +) + +func main() { + app := flags.NewApp("beacon light syncer tool") + app.Flags = []cli.Flag{ + utils.BeaconApiFlag, + utils.BeaconApiHeaderFlag, + utils.BeaconThresholdFlag, + utils.BeaconNoFilterFlag, + utils.BeaconConfigFlag, + utils.BeaconGenesisRootFlag, + utils.BeaconGenesisTimeFlag, + utils.BeaconCheckpointFlag, + //TODO datadir for optional permanent database + utils.MainnetFlag, + utils.SepoliaFlag, + utils.GoerliFlag, + utils.BlsyncApiFlag, + utils.BlsyncJWTSecretFlag, + verbosityFlag, + vmoduleFlag, + } + app.Action = sync + + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func sync(ctx *cli.Context) error { + usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" + output := io.Writer(os.Stderr) + if usecolor { + output = colorable.NewColorable(os.Stderr) + } + verbosity := log.FromLegacyLevel(ctx.Int(verbosityFlag.Name)) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(output, verbosity, usecolor))) + + headCh := make(chan types.ChainHeadEvent, 16) + client := blsync.NewClient(ctx) + sub := client.SubscribeChainHeadEvent(headCh) + go updateEngineApi(makeRPCClient(ctx), headCh) + client.Start() + // run until stopped + <-ctx.Done() + client.Stop() + sub.Unsubscribe() + close(headCh) + return nil +} + +func makeRPCClient(ctx *cli.Context) *rpc.Client { + if !ctx.IsSet(utils.BlsyncApiFlag.Name) { + log.Warn("No engine API target specified, performing a dry run") + return nil + } + if !ctx.IsSet(utils.BlsyncJWTSecretFlag.Name) { + utils.Fatalf("JWT secret parameter missing") //TODO use default if datadir is specified + } + + engineApiUrl, jwtFileName := ctx.String(utils.BlsyncApiFlag.Name), ctx.String(utils.BlsyncJWTSecretFlag.Name) + var jwtSecret [32]byte + if jwt, err := node.ObtainJWTSecret(jwtFileName); err == nil { + copy(jwtSecret[:], jwt) + } else { + utils.Fatalf("Error loading or generating JWT secret: %v", err) + } + auth := node.NewJWTAuth(jwtSecret) + cl, err := rpc.DialOptions(context.Background(), engineApiUrl, rpc.WithHTTPAuth(auth)) + if err != nil { + utils.Fatalf("Could not create RPC client: %v", err) + } + return cl +} diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 5f52f1df5442..37d17fb1e77d 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/accounts/usbwallet" + "github.com/ethereum/go-ethereum/beacon/blsync" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -221,6 +222,8 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { } catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon) stack.RegisterLifecycle(simBeacon) + } else if ctx.IsSet(utils.BeaconApiFlag.Name) { + stack.RegisterLifecycle(catalyst.NewBlsync(blsync.NewClient(ctx), eth)) } else { err := catalyst.Register(stack, eth) if err != nil { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 9a88e9f2e8b4..d79d23e22687 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -146,6 +146,14 @@ var ( configFileFlag, utils.LogDebugFlag, utils.LogBacktraceAtFlag, + utils.BeaconApiFlag, + utils.BeaconApiHeaderFlag, + utils.BeaconThresholdFlag, + utils.BeaconNoFilterFlag, + utils.BeaconConfigFlag, + utils.BeaconGenesisRootFlag, + utils.BeaconGenesisTimeFlag, + utils.BeaconCheckpointFlag, }, utils.NetworkFlags, utils.DatabaseFlags) rpcFlags = []cli.Flag{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index fad567cd55d2..e002975d53cf 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" + bparams "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/fdlimit" "github.com/ethereum/go-ethereum/core" @@ -281,6 +282,58 @@ var ( Value: ethconfig.Defaults.TransactionHistory, Category: flags.StateCategory, } + // Beacon client light sync settings + BeaconApiFlag = &cli.StringSliceFlag{ + Name: "beacon.api", + Usage: "Beacon node (CL) light client API URL. This flag can be given multiple times.", + Category: flags.BeaconCategory, + } + BeaconApiHeaderFlag = &cli.StringSliceFlag{ + Name: "beacon.api.header", + Usage: "Pass custom HTTP header fields to the emote beacon node API in \"key:value\" format. This flag can be given multiple times.", + Category: flags.BeaconCategory, + } + BeaconThresholdFlag = &cli.IntFlag{ + Name: "beacon.threshold", + Usage: "Beacon sync committee participation threshold", + Value: bparams.SyncCommitteeSupermajority, + Category: flags.BeaconCategory, + } + BeaconNoFilterFlag = &cli.BoolFlag{ + Name: "beacon.nofilter", + Usage: "Disable future slot signature filter", + Category: flags.BeaconCategory, + } + BeaconConfigFlag = &cli.StringFlag{ + Name: "beacon.config", + Usage: "Beacon chain config YAML file", + Category: flags.BeaconCategory, + } + BeaconGenesisRootFlag = &cli.StringFlag{ + Name: "beacon.genesis.gvroot", + Usage: "Beacon chain genesis validators root", + Category: flags.BeaconCategory, + } + BeaconGenesisTimeFlag = &cli.Uint64Flag{ + Name: "beacon.genesis.time", + Usage: "Beacon chain genesis time", + Category: flags.BeaconCategory, + } + BeaconCheckpointFlag = &cli.StringFlag{ + Name: "beacon.checkpoint", + Usage: "Beacon chain weak subjectivity checkpoint block hash", + Category: flags.BeaconCategory, + } + BlsyncApiFlag = &cli.StringFlag{ + Name: "blsync.engine.api", + Usage: "Target EL engine API URL", + Category: flags.BeaconCategory, + } + BlsyncJWTSecretFlag = &cli.StringFlag{ + Name: "blsync.jwtsecret", + Usage: "Path to a JWT secret to use for target engine API endpoint", + Category: flags.BeaconCategory, + } // Transaction pool settings TxPoolLocalsFlag = &cli.StringFlag{ Name: "txpool.locals", diff --git a/eth/catalyst/blsync.go b/eth/catalyst/blsync.go new file mode 100644 index 000000000000..4877cf4c6361 --- /dev/null +++ b/eth/catalyst/blsync.go @@ -0,0 +1,88 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" +) + +// Blsync tracks the head of the beacon chain through the beacon light client +// and drives the local node via ConsensusAPI. +type Blsync struct { + engine *ConsensusAPI + client Client + headCh chan types.ChainHeadEvent + headSub event.Subscription + + quitCh chan struct{} +} + +type Client interface { + SubscribeChainHeadEvent(ch chan<- types.ChainHeadEvent) event.Subscription + Start() + Stop() +} + +// NewBlsync creates a new beacon light syncer. +func NewBlsync(client Client, eth *eth.Ethereum) *Blsync { + return &Blsync{ + engine: newConsensusAPIWithoutHeartbeat(eth), + client: client, + headCh: make(chan types.ChainHeadEvent, 16), + quitCh: make(chan struct{}), + } +} + +// Start starts underlying beacon light client and the sync logic for driving +// the local node. +func (b *Blsync) Start() error { + log.Info("Beacon light sync started") + b.headSub = b.client.SubscribeChainHeadEvent(b.headCh) + go b.client.Start() + + for { + select { + case <-b.quitCh: + return nil + case head := <-b.headCh: + if _, err := b.engine.NewPayloadV2(*head.HeadBlock); err != nil { + log.Error("failed to send new payload", "err", err) + continue + } + update := engine.ForkchoiceStateV1{ + HeadBlockHash: head.HeadBlock.BlockHash, + SafeBlockHash: head.Finalized, //TODO pass finalized or empty hash here? + FinalizedBlockHash: head.Finalized, + } + if _, err := b.engine.ForkchoiceUpdatedV1(update, nil); err != nil { + log.Error("failed to send forkchoice updated", "err", err) + continue + } + } + } +} + +// Stop signals to the light client and syncer to exit. +func (b *Blsync) Stop() error { + b.client.Stop() + close(b.quitCh) + return nil +} diff --git a/go.mod b/go.mod index 6591bee62ff1..ca45364b8bf2 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,8 @@ require ( github.com/crate-crypto/go-kzg-4844 v0.7.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 - github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 + github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 + github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 github.com/ethereum/c-kzg-4844 v0.4.0 github.com/fatih/color v1.13.0 github.com/ferranbt/fastssz v0.1.2 @@ -54,6 +55,8 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 + github.com/protolambda/zrnt v0.30.0 + github.com/protolambda/ztyp v0.2.2 github.com/rs/cors v1.7.0 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.2.0 diff --git a/go.sum b/go.sum index cc74e15cb4b9..18236bf8e74d 100644 --- a/go.sum +++ b/go.sum @@ -149,9 +149,11 @@ github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwu github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao= +github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 h1:qwcF+vdFrvPSEUDSX5RVoRccG8a5DhOdWdQ4zN62zzo= -github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 h1:+3HCtB74++ClLy8GgjUQYeC8R4ILzVcIe8+5edAJJnE= +github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -239,6 +241,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -298,6 +301,7 @@ github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6w github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -380,6 +384,7 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= @@ -448,8 +453,14 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e/go.mod h1:MPZvj2Pr0N8/dXyTPS5REeg2sdLG7t8DRzC1rLv925w= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 h1:cZC+usqsYgHtlBaGulVnZ1hfKAi8iWtujBnRLQE698c= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7/go.mod h1:IToEjHuttnUzwZI5KBSM/LOOW3qLbbrHOEfp3SbECGY= +github.com/protolambda/messagediff v1.4.0/go.mod h1:LboJp0EwIbJsePYpzh5Op/9G1/4mIztMRYzzwR0dR2M= +github.com/protolambda/zrnt v0.30.0 h1:pHEn69ZgaDFGpLGGYG1oD7DvYI7RDirbMBPfbC+8p4g= +github.com/protolambda/zrnt v0.30.0/go.mod h1:qcdX9CXFeVNCQK/q0nswpzhd+31RHMk2Ax/2lMsJ4Jw= +github.com/protolambda/ztyp v0.2.2 h1:rVcL3vBu9W/aV646zF6caLS/dyn9BN8NYiuJzicLNyY= +github.com/protolambda/ztyp v0.2.2/go.mod h1:9bYgKGqg3wJqT9ac1gI2hnVb0STQq7p/1lapqrqY1dU= github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220714111606-acbb2962fb48 h1:cSo6/vk8YpvkLbk9v3FO97cakNmUoxwi2KMP8hd5WIw= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -842,6 +853,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/flags/categories.go b/internal/flags/categories.go index 3ff0767921b9..c044e28f384c 100644 --- a/internal/flags/categories.go +++ b/internal/flags/categories.go @@ -20,6 +20,7 @@ import "github.com/urfave/cli/v2" const ( EthCategory = "ETHEREUM" + BeaconCategory = "BEACON CHAIN" LightCategory = "LIGHT CLIENT" DevCategory = "DEVELOPER CHAIN" StateCategory = "STATE HISTORY MANAGEMENT" diff --git a/node/node.go b/node/node.go index dfa83d58c726..c5cb552d2737 100644 --- a/node/node.go +++ b/node/node.go @@ -339,15 +339,9 @@ func (n *Node) closeDataDir() { } } -// obtainJWTSecret loads the jwt-secret, either from the provided config, -// or from the default location. If neither of those are present, it generates -// a new secret and stores to the default location. -func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { - fileName := cliParam - if len(fileName) == 0 { - // no path provided, use default - fileName = n.ResolvePath(datadirJWTKey) - } +// ObtainJWTSecret loads the jwt-secret from the provided config. If the file is not +// present, it generates a new secret and stores to the given location. +func ObtainJWTSecret(fileName string) ([]byte, error) { // try reading from file if data, err := os.ReadFile(fileName); err == nil { jwtSecret := common.FromHex(strings.TrimSpace(string(data))) @@ -373,6 +367,18 @@ func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { return jwtSecret, nil } +// obtainJWTSecret loads the jwt-secret, either from the provided config, +// or from the default location. If neither of those are present, it generates +// a new secret and stores to the default location. +func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { + fileName := cliParam + if len(fileName) == 0 { + // no path provided, use default + fileName = n.ResolvePath(datadirJWTKey) + } + return ObtainJWTSecret(fileName) +} + // startRPC is a helper method to configure all the various RPC endpoints during node // startup. It's not meant to be called at any time afterwards as it makes certain // assumptions about the state of the node. From 3bebabbd036d4f550e32bb20a92bf7da6e6a2797 Mon Sep 17 00:00:00 2001 From: cuinix <65650185+cuinix@users.noreply.github.com> Date: Fri, 8 Mar 2024 05:25:08 +0800 Subject: [PATCH 69/91] accounts: remove redundant string conversion (#29184) --- accounts/accounts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/accounts/accounts.go b/accounts/accounts.go index 6c351a9649ea..b995498a6db9 100644 --- a/accounts/accounts.go +++ b/accounts/accounts.go @@ -195,7 +195,7 @@ func TextHash(data []byte) []byte { // // This gives context to the signed message and prevents signing of transactions. func TextAndHash(data []byte) ([]byte, string) { - msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), string(data)) + msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data) hasher := sha3.NewLegacyKeccak256() hasher.Write([]byte(msg)) return hasher.Sum(nil), msg From cd490608e344e388edd7ef3dd323968d706ccf8c Mon Sep 17 00:00:00 2001 From: hyhnet Date: Fri, 8 Mar 2024 05:56:19 +0800 Subject: [PATCH 70/91] all: fix typos in comments (#29186) --- cmd/evm/internal/t8ntool/block.go | 2 +- cmd/evm/internal/t8ntool/transaction.go | 2 +- cmd/evm/internal/t8ntool/transition.go | 2 +- cmd/evm/internal/t8ntool/tx_iterator.go | 2 +- cmd/evm/internal/t8ntool/utils.go | 2 +- core/types/account.go | 2 +- core/types/receipt_test.go | 4 ++-- crypto/kzg4844/kzg4844.go | 2 +- crypto/secp256k1/libsecp256k1/include/secp256k1.h | 2 +- crypto/secp256k1/libsecp256k1/sage/group_prover.sage | 2 +- eth/handler.go | 2 +- eth/protocols/snap/sync_test.go | 4 ++-- ethclient/gethclient/gethclient_test.go | 2 +- rpc/client.go | 2 +- 14 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/evm/internal/t8ntool/block.go b/cmd/evm/internal/t8ntool/block.go index a2dc4734372b..62c8593a1d47 100644 --- a/cmd/evm/internal/t8ntool/block.go +++ b/cmd/evm/internal/t8ntool/block.go @@ -242,7 +242,7 @@ func readInput(ctx *cli.Context) (*bbInput, error) { if headerStr == stdinSelector || ommersStr == stdinSelector || txsStr == stdinSelector || cliqueStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { - return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) } } if cliqueStr != stdinSelector && cliqueStr != "" { diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 8533b7863769..7f66ba4d85d6 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -86,7 +86,7 @@ func Transaction(ctx *cli.Context) error { if txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) } // Decode the body of already signed transactions body = common.FromHex(inputData.TxRlp) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 7802d4965199..aa0483a8ba67 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -135,7 +135,7 @@ func Transition(ctx *cli.Context) error { if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) } } if allocStr != stdinSelector { diff --git a/cmd/evm/internal/t8ntool/tx_iterator.go b/cmd/evm/internal/t8ntool/tx_iterator.go index 8f28dc70223b..046f62314dae 100644 --- a/cmd/evm/internal/t8ntool/tx_iterator.go +++ b/cmd/evm/internal/t8ntool/tx_iterator.go @@ -127,7 +127,7 @@ func loadTransactions(txStr string, inputData *input, env stEnv, chainConfig *pa return newRlpTxIterator(body), nil } if err := json.Unmarshal(data, &txsWithKeys); err != nil { - return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err)) + return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshalling txs-file: %v", err)) } } else { if len(inputData.TxRlp) > 0 { diff --git a/cmd/evm/internal/t8ntool/utils.go b/cmd/evm/internal/t8ntool/utils.go index 8ec38c7618de..42f5471e7b24 100644 --- a/cmd/evm/internal/t8ntool/utils.go +++ b/cmd/evm/internal/t8ntool/utils.go @@ -33,7 +33,7 @@ func readFile(path, desc string, dest interface{}) error { defer inFile.Close() decoder := json.NewDecoder(inFile) if err := decoder.Decode(dest); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling %s file: %v", desc, err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshalling %s file: %v", desc, err)) } return nil } diff --git a/core/types/account.go b/core/types/account.go index bb0f4ca02e10..52ce184cda5d 100644 --- a/core/types/account.go +++ b/core/types/account.go @@ -52,7 +52,7 @@ type accountMarshaling struct { } // storageJSON represents a 256 bit byte array, but allows less than 256 bits when -// unmarshaling from hex. +// unmarshalling from hex. type storageJSON common.Hash func (h *storageJSON) UnmarshalText(text []byte) error { diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index a7b26444712f..fc51eb11a528 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -343,7 +343,7 @@ func TestReceiptJSON(t *testing.T) { r := Receipt{} err = r.UnmarshalJSON(b) if err != nil { - t.Fatal("error unmarshaling receipt from json:", err) + t.Fatal("error unmarshalling receipt from json:", err) } } } @@ -360,7 +360,7 @@ func TestEffectiveGasPriceNotRequired(t *testing.T) { r2 := Receipt{} err = r2.UnmarshalJSON(b) if err != nil { - t.Fatal("error unmarshaling receipt from json:", err) + t.Fatal("error unmarshalling receipt from json:", err) } } diff --git a/crypto/kzg4844/kzg4844.go b/crypto/kzg4844/kzg4844.go index 52124df67461..168ff8347095 100644 --- a/crypto/kzg4844/kzg4844.go +++ b/crypto/kzg4844/kzg4844.go @@ -85,7 +85,7 @@ type Claim [32]byte var useCKZG atomic.Bool // UseCKZG can be called to switch the default Go implementation of KZG to the C -// library if fo some reason the user wishes to do so (e.g. consensus bug in one +// library if for some reason the user wishes to do so (e.g. consensus bug in one // or the other). func UseCKZG(use bool) error { if use && !ckzgAvailable { diff --git a/crypto/secp256k1/libsecp256k1/include/secp256k1.h b/crypto/secp256k1/libsecp256k1/include/secp256k1.h index f268e309d0bf..76af8396918e 100644 --- a/crypto/secp256k1/libsecp256k1/include/secp256k1.h +++ b/crypto/secp256k1/libsecp256k1/include/secp256k1.h @@ -357,7 +357,7 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact( /** Verify an ECDSA signature. * * Returns: 1: correct signature - * 0: incorrect or unparseable signature + * 0: incorrect or unparsable signature * Args: ctx: a secp256k1 context object, initialized for verification. * In: sig: the signature being verified (cannot be NULL) * msg32: the 32-byte message hash being verified (cannot be NULL) diff --git a/crypto/secp256k1/libsecp256k1/sage/group_prover.sage b/crypto/secp256k1/libsecp256k1/sage/group_prover.sage index ab580c5b23bb..68882e93659a 100644 --- a/crypto/secp256k1/libsecp256k1/sage/group_prover.sage +++ b/crypto/secp256k1/libsecp256k1/sage/group_prover.sage @@ -17,7 +17,7 @@ # - A constraint describing the requirements of the law, called "require" # * Implementations are transliterated into functions that operate as well on # algebraic input points, and are called once per combination of branches -# exectured. Each execution returns: +# executed. Each execution returns: # - A constraint describing the assumptions this implementation requires # (such as Z1=1), called "assumeFormula" # - A constraint describing the assumptions this specific branch requires, diff --git a/eth/handler.go b/eth/handler.go index a32a04e00b72..0d27e061c49b 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -497,7 +497,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { } // Send the transaction (if it's small enough) directly to a subset of // the peers that have not received it yet, ensuring that the flow of - // transactions is groupped by account to (try and) avoid nonce gaps. + // transactions is grouped by account to (try and) avoid nonce gaps. // // To do this, we hash the local enode IW with together with a peer's // enode ID together with the transaction sender and broadcast if diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index b780868b4e06..cea83aa2bc8e 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1502,7 +1502,7 @@ func getCodeByHash(hash common.Hash) []byte { return nil } -// makeAccountTrieNoStorage spits out a trie, along with the leafs +// makeAccountTrieNoStorage spits out a trie, along with the leaves func makeAccountTrieNoStorage(n int, scheme string) (string, *trie.Trie, []*kv) { var ( db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) @@ -1650,7 +1650,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots return db.Scheme(), accTrie, entries, storageTries, storageEntries } -// makeAccountTrieWithStorage spits out a trie, along with the leafs +// makeAccountTrieWithStorage spits out a trie, along with the leaves func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, boundary bool, uneven bool) (*trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 158886475eda..d562bcda1f01 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -146,7 +146,7 @@ func TestGethClient(t *testing.T) { func(t *testing.T) { testCallContractWithBlockOverrides(t, client) }, }, // The testaccesslist is a bit time-sensitive: the newTestBackend imports - // one block. The `testAcessList` fails if the miner has not yet created a + // one block. The `testAccessList` fails if the miner has not yet created a // new pending-block after the import event. // Hence: this test should be last, execute the tests serially. { diff --git a/rpc/client.go b/rpc/client.go index 2b0016db8f4e..eef6ee21cf05 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -70,7 +70,7 @@ type BatchElem struct { // discarded. Result interface{} // Error is set if the server returns an error for this request, or if - // unmarshaling into Result fails. It is not set for I/O errors. + // unmarshalling into Result fails. It is not set for I/O errors. Error error } From c41105ce80f12f60ec4bf6c65c4c59c6bf4a86e7 Mon Sep 17 00:00:00 2001 From: Sebastian Stammler Date: Fri, 8 Mar 2024 00:01:31 +0100 Subject: [PATCH 71/91] log: add Handler getter to Logger interface (#28793) log: Add Handler getter to Logger interface --- internal/testlog/testlog.go | 4 ++++ log/logger.go | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/internal/testlog/testlog.go b/internal/testlog/testlog.go index 037b7ee9c120..3cdbea6e0537 100644 --- a/internal/testlog/testlog.go +++ b/internal/testlog/testlog.go @@ -98,6 +98,10 @@ func LoggerWithHandler(t *testing.T, handler slog.Handler) log.Logger { } } +func (l *logger) Handler() slog.Handler { + return l.l.Handler() +} + func (l *logger) Write(level slog.Level, msg string, ctx ...interface{}) {} func (l *logger) Enabled(ctx context.Context, level slog.Level) bool { diff --git a/log/logger.go b/log/logger.go index 75e364304488..c28bbde56840 100644 --- a/log/logger.go +++ b/log/logger.go @@ -137,6 +137,9 @@ type Logger interface { // Enabled reports whether l emits log records at the given context and level. Enabled(ctx context.Context, level slog.Level) bool + + // Handler returns the underlying handler of the inner logger. + Handler() slog.Handler } type logger struct { @@ -150,6 +153,10 @@ func NewLogger(h slog.Handler) Logger { } } +func (l *logger) Handler() slog.Handler { + return l.inner.Handler() +} + // write logs a message at the specified level: func (l *logger) Write(level slog.Level, msg string, attrs ...any) { if !l.inner.Enabled(context.Background(), level) { From d35c8f0c25d3b5781e016252625b582c9553601a Mon Sep 17 00:00:00 2001 From: colin <102356659+colinlyguo@users.noreply.github.com> Date: Fri, 8 Mar 2024 19:13:46 +0800 Subject: [PATCH 72/91] ethclient/gethclient: add blob transaction fields in toCallArg (#29198) --- ethclient/gethclient/gethclient.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 73d05d499efe..b1678b67664e 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -245,6 +245,12 @@ func toCallArg(msg ethereum.CallMsg) interface{} { if msg.AccessList != nil { arg["accessList"] = msg.AccessList } + if msg.BlobGasFeeCap != nil { + arg["maxFeePerBlobGas"] = (*hexutil.Big)(msg.BlobGasFeeCap) + } + if msg.BlobHashes != nil { + arg["blobVersionedHashes"] = msg.BlobHashes + } return arg } From e31709db6570e302557a9bccd681034ea0dcc246 Mon Sep 17 00:00:00 2001 From: Haotian <51777534+tmelhao@users.noreply.github.com> Date: Fri, 8 Mar 2024 19:15:52 +0800 Subject: [PATCH 73/91] console: fix the wrong error msg of datadir testcase (#29183) --- console/console_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/console/console_test.go b/console/console_test.go index 4c30c1b49cc6..d210a993c2d7 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -152,8 +152,7 @@ func (env *tester) Close(t *testing.T) { } // Tests that the node lists the correct welcome message, notably that it contains -// the instance name, coinbase account, block number, data directory and supported -// console modules. +// the instance name, block number, data directory and supported console modules. func TestWelcome(t *testing.T) { tester := newTester(t, nil) defer tester.Close(t) @@ -171,7 +170,10 @@ func TestWelcome(t *testing.T) { t.Fatalf("console output missing sync status: have\n%s\nwant also %s", output, want) } if want := fmt.Sprintf("datadir: %s", tester.workspace); !strings.Contains(output, want) { - t.Fatalf("console output missing coinbase: have\n%s\nwant also %s", output, want) + t.Fatalf("console output missing datadir: have\n%s\nwant also %s", output, want) + } + if want := "modules: "; !strings.Contains(output, want) { + t.Fatalf("console output missing modules: have\n%s\nwant also %s", output, want) } } From 3dc549b3d75af790e78ef2d7f63a947efb9b0e95 Mon Sep 17 00:00:00 2001 From: Kero Date: Mon, 11 Mar 2024 03:01:26 +0800 Subject: [PATCH 74/91] p2p/simulations/adapters: fix error messages in TestTCPPipeBidirections (#29207) --- p2p/simulations/adapters/inproc_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/simulations/adapters/inproc_test.go b/p2p/simulations/adapters/inproc_test.go index 2a61508fe18b..d0539ca86752 100644 --- a/p2p/simulations/adapters/inproc_test.go +++ b/p2p/simulations/adapters/inproc_test.go @@ -78,7 +78,7 @@ func TestTCPPipeBidirections(t *testing.T) { } if !bytes.Equal(expected, out) { - t.Fatalf("expected %#v, got %#v", out, expected) + t.Fatalf("expected %#v, got %#v", expected, out) } else { msg := []byte(fmt.Sprintf("pong %02d", i)) if _, err := c2.Write(msg); err != nil { @@ -94,7 +94,7 @@ func TestTCPPipeBidirections(t *testing.T) { t.Fatal(err) } if !bytes.Equal(expected, out) { - t.Fatalf("expected %#v, got %#v", out, expected) + t.Fatalf("expected %#v, got %#v", expected, out) } } } From b393ad8d29fe002fe6c0329a09d7715b00030c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 11 Mar 2024 10:06:57 +0200 Subject: [PATCH 75/91] cmd, core, metrics: always report expensive metrics (#29191) * cmd, core, metrics: always report expensive metrics * core, metrics: report block processing metrics as resetting timer * metrics: update reporter tests --- cmd/geth/config.go | 2 +- cmd/utils/flags.go | 6 -- cmd/utils/flags_legacy.go | 29 ++++--- core/blockchain.go | 40 +++++----- core/state/state_object.go | 26 +++--- core/state/statedb.go | 87 +++++++++------------ internal/flags/categories.go | 1 - metrics/config.go | 2 +- metrics/influxdb/influxdb.go | 25 +++--- metrics/influxdb/testdata/influxdbv1.want | 2 +- metrics/influxdb/testdata/influxdbv2.want | 2 +- metrics/metrics.go | 25 ------ metrics/prometheus/collector.go | 9 ++- metrics/prometheus/testdata/prometheus.want | 5 +- 14 files changed, 112 insertions(+), 149 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 37d17fb1e77d..cf4cdef76ce1 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -267,7 +267,7 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { cfg.Metrics.Enabled = ctx.Bool(utils.MetricsEnabledFlag.Name) } if ctx.IsSet(utils.MetricsEnabledExpensiveFlag.Name) { - cfg.Metrics.EnabledExpensive = ctx.Bool(utils.MetricsEnabledExpensiveFlag.Name) + log.Warn("Expensive metrics are collected by default, please remove this flag", "flag", utils.MetricsEnabledExpensiveFlag.Name) } if ctx.IsSet(utils.MetricsHTTPFlag.Name) { cfg.Metrics.HTTP = ctx.String(utils.MetricsHTTPFlag.Name) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e002975d53cf..b38f33b8dd7f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -862,12 +862,6 @@ var ( Usage: "Enable metrics collection and reporting", Category: flags.MetricsCategory, } - MetricsEnabledExpensiveFlag = &cli.BoolFlag{ - Name: "metrics.expensive", - Usage: "Enable expensive metrics collection and reporting", - Category: flags.MetricsCategory, - } - // MetricsHTTPFlag defines the endpoint for a stand-alone metrics HTTP endpoint. // Since the pprof service enables sensitive/vulnerable behavior, this allows a user // to enable a public-OK metrics endpoint without having to worry about ALSO exposing diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 49321053c672..1dfd1a5f8934 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -93,35 +93,35 @@ var ( Name: "light.serve", Usage: "Maximum percentage of time allowed for serving LES requests (deprecated)", Value: ethconfig.Defaults.LightServ, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightIngressFlag = &cli.IntFlag{ Name: "light.ingress", Usage: "Incoming bandwidth limit for serving light clients (deprecated)", Value: ethconfig.Defaults.LightIngress, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightEgressFlag = &cli.IntFlag{ Name: "light.egress", Usage: "Outgoing bandwidth limit for serving light clients (deprecated)", Value: ethconfig.Defaults.LightEgress, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightMaxPeersFlag = &cli.IntFlag{ Name: "light.maxpeers", Usage: "Maximum number of light clients to serve, or light servers to attach to (deprecated)", Value: ethconfig.Defaults.LightPeers, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightNoPruneFlag = &cli.BoolFlag{ Name: "light.nopruning", Usage: "Disable ancient light chain data pruning (deprecated)", - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightNoSyncServeFlag = &cli.BoolFlag{ Name: "light.nosyncserve", Usage: "Enables serving light clients before syncing (deprecated)", - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } // Deprecated November 2023 LogBacktraceAtFlag = &cli.StringFlag{ @@ -138,19 +138,24 @@ var ( // Deprecated February 2024 MinerNewPayloadTimeoutFlag = &cli.DurationFlag{ Name: "miner.newpayload-timeout", - Usage: "Specify the maximum time allowance for creating a new payload", + Usage: "Specify the maximum time allowance for creating a new payload (deprecated)", Value: ethconfig.Defaults.Miner.Recommit, - Category: flags.MinerCategory, + Category: flags.DeprecatedCategory, } MinerEtherbaseFlag = &cli.StringFlag{ Name: "miner.etherbase", - Usage: "0x prefixed public address for block mining rewards", - Category: flags.MinerCategory, + Usage: "0x prefixed public address for block mining rewards (deprecated)", + Category: flags.DeprecatedCategory, } MiningEnabledFlag = &cli.BoolFlag{ Name: "mine", - Usage: "Enable mining", - Category: flags.MinerCategory, + Usage: "Enable mining (deprecated)", + Category: flags.DeprecatedCategory, + } + MetricsEnabledExpensiveFlag = &cli.BoolFlag{ + Name: "metrics.expensive", + Usage: "Enable expensive metrics collection and reporting (deprecated)", + Category: flags.DeprecatedCategory, } ) diff --git a/core/blockchain.go b/core/blockchain.go index 67b49cfe0262..9bd7fdcd9544 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -61,26 +61,26 @@ var ( chainInfoGauge = metrics.NewRegisteredGaugeInfo("chain/info", nil) - accountReadTimer = metrics.NewRegisteredTimer("chain/account/reads", nil) - accountHashTimer = metrics.NewRegisteredTimer("chain/account/hashes", nil) - accountUpdateTimer = metrics.NewRegisteredTimer("chain/account/updates", nil) - accountCommitTimer = metrics.NewRegisteredTimer("chain/account/commits", nil) - - storageReadTimer = metrics.NewRegisteredTimer("chain/storage/reads", nil) - storageHashTimer = metrics.NewRegisteredTimer("chain/storage/hashes", nil) - storageUpdateTimer = metrics.NewRegisteredTimer("chain/storage/updates", nil) - storageCommitTimer = metrics.NewRegisteredTimer("chain/storage/commits", nil) - - snapshotAccountReadTimer = metrics.NewRegisteredTimer("chain/snapshot/account/reads", nil) - snapshotStorageReadTimer = metrics.NewRegisteredTimer("chain/snapshot/storage/reads", nil) - snapshotCommitTimer = metrics.NewRegisteredTimer("chain/snapshot/commits", nil) - - triedbCommitTimer = metrics.NewRegisteredTimer("chain/triedb/commits", nil) - - blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) - blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil) - blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) - blockWriteTimer = metrics.NewRegisteredTimer("chain/write", nil) + accountReadTimer = metrics.NewRegisteredResettingTimer("chain/account/reads", nil) + accountHashTimer = metrics.NewRegisteredResettingTimer("chain/account/hashes", nil) + accountUpdateTimer = metrics.NewRegisteredResettingTimer("chain/account/updates", nil) + accountCommitTimer = metrics.NewRegisteredResettingTimer("chain/account/commits", nil) + + storageReadTimer = metrics.NewRegisteredResettingTimer("chain/storage/reads", nil) + storageHashTimer = metrics.NewRegisteredResettingTimer("chain/storage/hashes", nil) + storageUpdateTimer = metrics.NewRegisteredResettingTimer("chain/storage/updates", nil) + storageCommitTimer = metrics.NewRegisteredResettingTimer("chain/storage/commits", nil) + + snapshotAccountReadTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/account/reads", nil) + snapshotStorageReadTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/storage/reads", nil) + snapshotCommitTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/commits", nil) + + triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) + + blockInsertTimer = metrics.NewRegisteredResettingTimer("chain/inserts", nil) + blockValidationTimer = metrics.NewRegisteredResettingTimer("chain/validation", nil) + blockExecutionTimer = metrics.NewRegisteredResettingTimer("chain/execution", nil) + blockWriteTimer = metrics.NewRegisteredResettingTimer("chain/write", nil) blockReorgMeter = metrics.NewRegisteredMeter("chain/reorg/executes", nil) blockReorgAddMeter = metrics.NewRegisteredMeter("chain/reorg/add", nil) diff --git a/core/state/state_object.go b/core/state/state_object.go index fc26af68dbe7..6dea68465baa 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/holiman/uint256" @@ -197,9 +196,8 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { if s.db.snap != nil { start := time.Now() enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key.Bytes())) - if metrics.EnabledExpensive { - s.db.SnapshotStorageReads += time.Since(start) - } + s.db.SnapshotStorageReads += time.Since(start) + if len(enc) > 0 { _, content, _, err := rlp.Split(enc) if err != nil { @@ -217,9 +215,8 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { return common.Hash{} } val, err := tr.GetStorage(s.address, key.Bytes()) - if metrics.EnabledExpensive { - s.db.StorageReads += time.Since(start) - } + s.db.StorageReads += time.Since(start) + if err != nil { s.db.setError(err) return common.Hash{} @@ -283,9 +280,8 @@ func (s *stateObject) updateTrie() (Trie, error) { return s.trie, nil } // Track the amount of time wasted on updating the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now()) + // The snapshot storage map for the object var ( storage map[common.Hash][]byte @@ -370,9 +366,8 @@ func (s *stateObject) updateRoot() { return } // Track the amount of time wasted on hashing the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now()) + s.data.Root = tr.Hash() } @@ -386,9 +381,8 @@ func (s *stateObject) commit() (*trienode.NodeSet, error) { return nil, nil } // Track the amount of time wasted on committing the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) + // The trie is currently in an open state and could potentially contain // cached mutations. Call commit to acquire a set of nodes that have been // modified, the set can be nil if nothing to commit. diff --git a/core/state/statedb.go b/core/state/statedb.go index 4d1163d3c6af..f90b30f3994e 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" @@ -495,9 +494,8 @@ func (s *StateDB) GetTransientState(addr common.Address, key common.Hash) common // updateStateObject writes the given object to the trie. func (s *StateDB) updateStateObject(obj *stateObject) { // Track the amount of time wasted on updating the account from the trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) + // Encode the account and update the account trie addr := obj.Address() if err := s.trie.UpdateAccount(addr, &obj.data); err != nil { @@ -527,9 +525,8 @@ func (s *StateDB) updateStateObject(obj *stateObject) { // deleteStateObject removes the given object from the state trie. func (s *StateDB) deleteStateObject(obj *stateObject) { // Track the amount of time wasted on deleting the account from the trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) + // Delete the account from the trie addr := obj.Address() if err := s.trie.DeleteAccount(addr); err != nil { @@ -561,9 +558,8 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { if s.snap != nil { start := time.Now() acc, err := s.snap.Account(crypto.HashData(s.hasher, addr.Bytes())) - if metrics.EnabledExpensive { - s.SnapshotAccountReads += time.Since(start) - } + s.SnapshotAccountReads += time.Since(start) + if err == nil { if acc == nil { return nil @@ -587,9 +583,8 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { start := time.Now() var err error data, err = s.trie.GetAccount(addr) - if metrics.EnabledExpensive { - s.AccountReads += time.Since(start) - } + s.AccountReads += time.Since(start) + if err != nil { s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %w", addr.Bytes(), err)) return nil @@ -917,9 +912,8 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { s.stateObjectsPending = make(map[common.Address]struct{}) } // Track the amount of time wasted on hashing the account trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) + return s.trie.Hash() } @@ -1042,16 +1036,16 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root if err != nil { return nil, nil, err } - if metrics.EnabledExpensive { - n := int64(len(slots)) + // Report the metrics + n := int64(len(slots)) - slotDeletionMaxCount.UpdateIfGt(int64(len(slots))) - slotDeletionMaxSize.UpdateIfGt(int64(size)) + slotDeletionMaxCount.UpdateIfGt(int64(len(slots))) + slotDeletionMaxSize.UpdateIfGt(int64(size)) + + slotDeletionTimer.UpdateSince(start) + slotDeletionCount.Mark(n) + slotDeletionSize.Mark(int64(size)) - slotDeletionTimer.UpdateSince(start) - slotDeletionCount.Mark(n) - slotDeletionSize.Mark(int64(size)) - } return slots, nodes, nil } @@ -1190,10 +1184,8 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er } } // Write the account trie changes, measuring the amount of wasted time - var start time.Time - if metrics.EnabledExpensive { - start = time.Now() - } + start := time.Now() + root, set, err := s.trie.Commit(true) if err != nil { return common.Hash{}, err @@ -1205,23 +1197,23 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er } accountTrieNodesUpdated, accountTrieNodesDeleted = set.Size() } - if metrics.EnabledExpensive { - s.AccountCommits += time.Since(start) + // Report the commit metrics + s.AccountCommits += time.Since(start) + + accountUpdatedMeter.Mark(int64(s.AccountUpdated)) + storageUpdatedMeter.Mark(int64(s.StorageUpdated)) + accountDeletedMeter.Mark(int64(s.AccountDeleted)) + storageDeletedMeter.Mark(int64(s.StorageDeleted)) + accountTrieUpdatedMeter.Mark(int64(accountTrieNodesUpdated)) + accountTrieDeletedMeter.Mark(int64(accountTrieNodesDeleted)) + storageTriesUpdatedMeter.Mark(int64(storageTrieNodesUpdated)) + storageTriesDeletedMeter.Mark(int64(storageTrieNodesDeleted)) + s.AccountUpdated, s.AccountDeleted = 0, 0 + s.StorageUpdated, s.StorageDeleted = 0, 0 - accountUpdatedMeter.Mark(int64(s.AccountUpdated)) - storageUpdatedMeter.Mark(int64(s.StorageUpdated)) - accountDeletedMeter.Mark(int64(s.AccountDeleted)) - storageDeletedMeter.Mark(int64(s.StorageDeleted)) - accountTrieUpdatedMeter.Mark(int64(accountTrieNodesUpdated)) - accountTrieDeletedMeter.Mark(int64(accountTrieNodesDeleted)) - storageTriesUpdatedMeter.Mark(int64(storageTrieNodesUpdated)) - storageTriesDeletedMeter.Mark(int64(storageTrieNodesDeleted)) - s.AccountUpdated, s.AccountDeleted = 0, 0 - s.StorageUpdated, s.StorageDeleted = 0, 0 - } // If snapshotting is enabled, update the snapshot tree with this new version if s.snap != nil { - start := time.Now() + start = time.Now() // Only update if there's a state transition (skip empty Clique blocks) if parent := s.snap.Root(); parent != root { if err := s.snaps.Update(root, parent, s.convertAccountSet(s.stateObjectsDestruct), s.accounts, s.storages); err != nil { @@ -1235,9 +1227,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er log.Warn("Failed to cap snapshot tree", "root", root, "layers", 128, "err", err) } } - if metrics.EnabledExpensive { - s.SnapshotCommits += time.Since(start) - } + s.SnapshotCommits += time.Since(start) s.snap = nil } if root == (common.Hash{}) { @@ -1248,15 +1238,14 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er origin = types.EmptyRootHash } if root != origin { - start := time.Now() + start = time.Now() set := triestate.New(s.accountsOrigin, s.storagesOrigin) if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil { return common.Hash{}, err } s.originalRoot = root - if metrics.EnabledExpensive { - s.TrieDBCommits += time.Since(start) - } + s.TrieDBCommits += time.Since(start) + if s.onCommit != nil { s.onCommit(set) } diff --git a/internal/flags/categories.go b/internal/flags/categories.go index c044e28f384c..d426add55b10 100644 --- a/internal/flags/categories.go +++ b/internal/flags/categories.go @@ -21,7 +21,6 @@ import "github.com/urfave/cli/v2" const ( EthCategory = "ETHEREUM" BeaconCategory = "BEACON CHAIN" - LightCategory = "LIGHT CLIENT" DevCategory = "DEVELOPER CHAIN" StateCategory = "STATE HISTORY MANAGEMENT" TxPoolCategory = "TRANSACTION POOL (EVM)" diff --git a/metrics/config.go b/metrics/config.go index 2eb09fb48a33..72f94dd194c9 100644 --- a/metrics/config.go +++ b/metrics/config.go @@ -19,7 +19,7 @@ package metrics // Config contains the configuration for the metric collection. type Config struct { Enabled bool `toml:",omitempty"` - EnabledExpensive bool `toml:",omitempty"` + EnabledExpensive bool `toml:"-"` HTTP string `toml:",omitempty"` Port int `toml:",omitempty"` EnableInfluxDB bool `toml:",omitempty"` diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index bbc4fc024b34..5c8501fd9db5 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -98,20 +98,23 @@ func readMeter(namespace, name string, i interface{}) (string, map[string]interf } return measurement, fields case metrics.ResettingTimer: - t := metric.Snapshot() - if t.Count() == 0 { + ms := metric.Snapshot() + if ms.Count() == 0 { break } - ps := t.Percentiles([]float64{0.50, 0.95, 0.99}) - measurement := fmt.Sprintf("%s%s.span", namespace, name) + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + measurement := fmt.Sprintf("%s%s.timer", namespace, name) fields := map[string]interface{}{ - "count": t.Count(), - "max": t.Max(), - "mean": t.Mean(), - "min": t.Min(), - "p50": int(ps[0]), - "p95": int(ps[1]), - "p99": int(ps[2]), + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], } return measurement, fields } diff --git a/metrics/influxdb/testdata/influxdbv1.want b/metrics/influxdb/testdata/influxdbv1.want index 9443faedc5a2..ded9434c7314 100644 --- a/metrics/influxdb/testdata/influxdbv1.want +++ b/metrics/influxdb/testdata/influxdbv1.want @@ -7,5 +7,5 @@ goth.test/gauge_float64.gauge value=34567.89 978307200000000000 goth.test/gauge_info.gauge value="{\"arch\":\"amd64\",\"commit\":\"7caa2d8163ae3132c1c2d6978c76610caee2d949\",\"os\":\"linux\",\"protocol_versions\":\"64 65 66\",\"version\":\"1.10.18-unstable\"}" 978307200000000000 goth.test/histogram.histogram count=3i,max=3i,mean=2,min=1i,p25=1,p50=2,p75=3,p95=3,p99=3,p999=3,p9999=3,stddev=0.816496580927726,variance=0.6666666666666666 978307200000000000 goth.test/meter.meter count=0i,m1=0,m15=0,m5=0,mean=0 978307200000000000 -goth.test/resetting_timer.span count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000i,p95=120000000i,p99=120000000i 978307200000000000 +goth.test/resetting_timer.timer count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000,p75=40500000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000 978307200000000000 goth.test/timer.timer count=6i,m1=0,m15=0,m5=0,max=120000000i,mean=38333333.333333336,meanrate=0,min=20000000i,p50=22500000,p75=48000000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000,stddev=36545253.529775314,variance=1335555555555555.2 978307200000000000 diff --git a/metrics/influxdb/testdata/influxdbv2.want b/metrics/influxdb/testdata/influxdbv2.want index 9443faedc5a2..ded9434c7314 100644 --- a/metrics/influxdb/testdata/influxdbv2.want +++ b/metrics/influxdb/testdata/influxdbv2.want @@ -7,5 +7,5 @@ goth.test/gauge_float64.gauge value=34567.89 978307200000000000 goth.test/gauge_info.gauge value="{\"arch\":\"amd64\",\"commit\":\"7caa2d8163ae3132c1c2d6978c76610caee2d949\",\"os\":\"linux\",\"protocol_versions\":\"64 65 66\",\"version\":\"1.10.18-unstable\"}" 978307200000000000 goth.test/histogram.histogram count=3i,max=3i,mean=2,min=1i,p25=1,p50=2,p75=3,p95=3,p99=3,p999=3,p9999=3,stddev=0.816496580927726,variance=0.6666666666666666 978307200000000000 goth.test/meter.meter count=0i,m1=0,m15=0,m5=0,mean=0 978307200000000000 -goth.test/resetting_timer.span count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000i,p95=120000000i,p99=120000000i 978307200000000000 +goth.test/resetting_timer.timer count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000,p75=40500000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000 978307200000000000 goth.test/timer.timer count=6i,m1=0,m15=0,m5=0,max=120000000i,mean=38333333.333333336,meanrate=0,min=20000000i,p50=22500000,p75=48000000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000,stddev=36545253.529775314,variance=1335555555555555.2 978307200000000000 diff --git a/metrics/metrics.go b/metrics/metrics.go index 9ca8f115c0f7..9e0ac23dd511 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -24,23 +24,12 @@ import ( // for less cluttered pprof profiles. var Enabled = false -// EnabledExpensive is a soft-flag meant for external packages to check if costly -// metrics gathering is allowed or not. The goal is to separate standard metrics -// for health monitoring and debug metrics that might impact runtime performance. -var EnabledExpensive = false - // enablerFlags is the CLI flag names to use to enable metrics collections. var enablerFlags = []string{"metrics"} // enablerEnvVars is the env var names to use to enable metrics collections. var enablerEnvVars = []string{"GETH_METRICS"} -// expensiveEnablerFlags is the CLI flag names to use to enable metrics collections. -var expensiveEnablerFlags = []string{"metrics.expensive"} - -// expensiveEnablerEnvVars is the env var names to use to enable metrics collections. -var expensiveEnablerEnvVars = []string{"GETH_METRICS_EXPENSIVE"} - // Init enables or disables the metrics system. Since we need this to run before // any other code gets to create meters and timers, we'll actually do an ugly hack // and peek into the command line args for the metrics flag. @@ -53,14 +42,6 @@ func init() { } } } - for _, enabler := range expensiveEnablerEnvVars { - if val, found := syscall.Getenv(enabler); found && !EnabledExpensive { - if enable, _ := strconv.ParseBool(val); enable { // ignore error, flag parser will choke on it later - log.Info("Enabling expensive metrics collection") - EnabledExpensive = true - } - } - } for _, arg := range os.Args { flag := strings.TrimLeft(arg, "-") @@ -70,12 +51,6 @@ func init() { Enabled = true } } - for _, enabler := range expensiveEnablerFlags { - if !EnabledExpensive && flag == enabler { - log.Info("Enabling expensive metrics collection") - EnabledExpensive = true - } - } } } diff --git a/metrics/prometheus/collector.go b/metrics/prometheus/collector.go index 25b258d56ab1..353336763b0b 100644 --- a/metrics/prometheus/collector.go +++ b/metrics/prometheus/collector.go @@ -125,12 +125,13 @@ func (c *collector) addResettingTimer(name string, m metrics.ResettingTimerSnaps if m.Count() <= 0 { return } - ps := m.Percentiles([]float64{0.50, 0.95, 0.99}) + pv := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999} + ps := m.Percentiles(pv) c.writeSummaryCounter(name, m.Count()) c.buff.WriteString(fmt.Sprintf(typeSummaryTpl, mutateKey(name))) - c.writeSummaryPercentile(name, "0.50", ps[0]) - c.writeSummaryPercentile(name, "0.95", ps[1]) - c.writeSummaryPercentile(name, "0.99", ps[2]) + for i := range pv { + c.writeSummaryPercentile(name, strconv.FormatFloat(pv[i], 'f', -1, 64), ps[i]) + } c.buff.WriteRune('\n') } diff --git a/metrics/prometheus/testdata/prometheus.want b/metrics/prometheus/testdata/prometheus.want index 861c5f5cf087..a999d83801c6 100644 --- a/metrics/prometheus/testdata/prometheus.want +++ b/metrics/prometheus/testdata/prometheus.want @@ -53,9 +53,12 @@ test_meter 0 test_resetting_timer_count 6 # TYPE test_resetting_timer summary -test_resetting_timer {quantile="0.50"} 1.25e+07 +test_resetting_timer {quantile="0.5"} 1.25e+07 +test_resetting_timer {quantile="0.75"} 4.05e+07 test_resetting_timer {quantile="0.95"} 1.2e+08 test_resetting_timer {quantile="0.99"} 1.2e+08 +test_resetting_timer {quantile="0.999"} 1.2e+08 +test_resetting_timer {quantile="0.9999"} 1.2e+08 # TYPE test_timer_count counter test_timer_count 6 From 00c21128ef62be54bef798f3220f79ae2297be66 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 11 Mar 2024 05:05:17 -0500 Subject: [PATCH 76/91] core/txpool/blobpool: return ErrAlreadyKnown for duplicate txs (#29210) Signed-off-by: Lee Bousfield --- core/txpool/blobpool/blobpool.go | 6 +++++- core/txpool/blobpool/blobpool_test.go | 11 ++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 3ed698c1b18f..6dbcc9dadc05 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1131,8 +1131,12 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error { next = p.state.GetNonce(from) ) if uint64(len(p.index[from])) > tx.Nonce()-next { - // Account can support the replacement, but the price bump must also be met prev := p.index[from][int(tx.Nonce()-next)] + // Ensure the transaction is different than the one tracked locally + if prev.hash == tx.Hash() { + return txpool.ErrAlreadyKnown + } + // Account can support the replacement, but the price bump must also be met switch { case tx.GasFeeCapIntCmp(prev.execFeeCap.ToBig()) <= 0: return fmt.Errorf("%w: new tx gas fee cap %v <= %v queued", txpool.ErrReplaceUnderpriced, tx.GasFeeCap(), prev.execFeeCap) diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index f7644c1d0ab6..bac239db4769 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -984,9 +984,14 @@ func TestAdd(t *testing.T) { }, }, adds: []addtx{ - { // New account, 1 tx pending: reject replacement nonce 0 (ignore price for now) + { // New account, 1 tx pending: reject duplicate nonce 0 from: "alice", tx: makeUnsignedTx(0, 1, 1, 1), + err: txpool.ErrAlreadyKnown, + }, + { // New account, 1 tx pending: reject replacement nonce 0 (ignore price for now) + from: "alice", + tx: makeUnsignedTx(0, 1, 1, 2), err: txpool.ErrReplaceUnderpriced, }, { // New account, 1 tx pending: accept nonce 1 @@ -1009,10 +1014,10 @@ func TestAdd(t *testing.T) { tx: makeUnsignedTx(3, 1, 1, 1), err: nil, }, - { // Old account, 1 tx in chain, 1 tx pending: reject replacement nonce 1 (ignore price for now) + { // Old account, 1 tx in chain, 1 tx pending: reject duplicate nonce 1 from: "bob", tx: makeUnsignedTx(1, 1, 1, 1), - err: txpool.ErrReplaceUnderpriced, + err: txpool.ErrAlreadyKnown, }, { // Old account, 1 tx in chain, 1 tx pending: accept nonce 2 (ignore price for now) from: "bob", From fa4ade8ecb4e37687b464fdab6986c01cc1e50c2 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Mon, 11 Mar 2024 11:05:48 +0100 Subject: [PATCH 77/91] core: fix deprecation comment for GenesisAccount (#29218) core: fix deprecation comment --- core/genesis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/genesis.go b/core/genesis.go index 54570ac61e4c..3f1fde8dfca4 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -45,7 +45,7 @@ import ( var errGenesisNoConfig = errors.New("genesis has no chain configuration") -// Deprecated: use types.GenesisAccount instead. +// Deprecated: use types.Account instead. type GenesisAccount = types.Account // Deprecated: use types.GenesisAlloc instead. From ebf9e11af2ff701d0961623e817d37b421b96802 Mon Sep 17 00:00:00 2001 From: guangwu Date: Mon, 11 Mar 2024 18:17:16 +0800 Subject: [PATCH 78/91] beacon/light/request: fix typos (#29216) --- beacon/light/request/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon/light/request/server.go b/beacon/light/request/server.go index 999f64178af4..407eb69f497e 100644 --- a/beacon/light/request/server.go +++ b/beacon/light/request/server.go @@ -257,7 +257,7 @@ func (s *serverWithLimits) init() { } // subscribe subscribes to events which include parent (serverWithTimeout) events -// plus EvCanRequstAgain. +// plus EvCanRequestAgain. func (s *serverWithLimits) subscribe(eventCallback func(event Event)) { s.lock.Lock() defer s.lock.Unlock() @@ -415,7 +415,7 @@ func (s *serverWithLimits) delay(delay time.Duration) { } // fail reports that a response from the server was found invalid by the processing -// Module, disabling new requests for a dynamically adjused time period. +// Module, disabling new requests for a dynamically adjusted time period. func (s *serverWithLimits) fail(desc string) { s.lock.Lock() defer s.lock.Unlock() From 4e1116f9c513961b62dff146a7cce069fe7a36b0 Mon Sep 17 00:00:00 2001 From: San Ye Date: Tue, 12 Mar 2024 16:49:53 +0800 Subject: [PATCH 79/91] crypto/bn256/cloudflare: fix noescape-directive (#29222) --- crypto/bn256/cloudflare/gfp_decl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/bn256/cloudflare/gfp_decl.go b/crypto/bn256/cloudflare/gfp_decl.go index cf7f5654239f..1954d14a4a5a 100644 --- a/crypto/bn256/cloudflare/gfp_decl.go +++ b/crypto/bn256/cloudflare/gfp_decl.go @@ -13,7 +13,7 @@ import ( //nolint:varcheck,unused,deadcode var hasBMI2 = cpu.X86.HasBMI2 -// go:noescape +//go:noescape func gfpNeg(c, a *gfP) //go:noescape From 89cefe240fd22b01e413786e18ad35263c93a61f Mon Sep 17 00:00:00 2001 From: Bin <49082129+songzhibin97@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:00:34 +0800 Subject: [PATCH 80/91] cmd: use package filepath over path for file system operations (#29227) Package filepath implements utility routines for manipulating filename paths in a way compatible with the target operating system-defined file paths. Package path implements utility routines for manipulating slash-separated paths. The path package should only be used for paths separated by forward slashes, such as the paths in URLs --- cmd/devp2p/internal/ethtest/chain.go | 10 +++++----- cmd/devp2p/internal/ethtest/engine.go | 4 ++-- cmd/devp2p/internal/ethtest/suite_test.go | 4 ++-- cmd/era/main.go | 6 +++--- cmd/evm/internal/t8ntool/transition.go | 8 ++++---- cmd/utils/cmd.go | 12 ++++++------ cmd/utils/history_test.go | 6 +++--- core/blockchain_repair_test.go | 6 +++--- core/blockchain_snapshot_test.go | 4 ++-- core/rawdb/freezer_test.go | 3 ++- 10 files changed, 32 insertions(+), 31 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index e8b3725b17a5..a34a41dacdaa 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -26,7 +26,7 @@ import ( "io" "math/big" "os" - "path" + "path/filepath" "sort" "strings" @@ -56,21 +56,21 @@ type Chain struct { // NewChain takes the given chain.rlp file, and decodes and returns // the blocks from the file. func NewChain(dir string) (*Chain, error) { - gen, err := loadGenesis(path.Join(dir, "genesis.json")) + gen, err := loadGenesis(filepath.Join(dir, "genesis.json")) if err != nil { return nil, err } gblock := gen.ToBlock() - blocks, err := blocksFromFile(path.Join(dir, "chain.rlp"), gblock) + blocks, err := blocksFromFile(filepath.Join(dir, "chain.rlp"), gblock) if err != nil { return nil, err } - state, err := readState(path.Join(dir, "headstate.json")) + state, err := readState(filepath.Join(dir, "headstate.json")) if err != nil { return nil, err } - accounts, err := readAccounts(path.Join(dir, "accounts.json")) + accounts, err := readAccounts(filepath.Join(dir, "accounts.json")) if err != nil { return nil, err } diff --git a/cmd/devp2p/internal/ethtest/engine.go b/cmd/devp2p/internal/ethtest/engine.go index ea4fc76e6ff7..0e94efa5bdac 100644 --- a/cmd/devp2p/internal/ethtest/engine.go +++ b/cmd/devp2p/internal/ethtest/engine.go @@ -22,7 +22,7 @@ import ( "io" "net/http" "os" - "path" + "path/filepath" "time" "github.com/ethereum/go-ethereum/common" @@ -38,7 +38,7 @@ type EngineClient struct { // NewEngineClient creates a new engine client. func NewEngineClient(dir, url, jwt string) (*EngineClient, error) { - headfcu, err := os.ReadFile(path.Join(dir, "headfcu.json")) + headfcu, err := os.ReadFile(filepath.Join(dir, "headfcu.json")) if err != nil { return nil, fmt.Errorf("failed to read headfcu: %w", err) } diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index ad73bc9f90e7..d70adda51f92 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -20,7 +20,7 @@ import ( crand "crypto/rand" "fmt" "os" - "path" + "path/filepath" "testing" "time" @@ -39,7 +39,7 @@ func makeJWTSecret() (string, [32]byte, error) { if _, err := crand.Read(secret[:]); err != nil { return "", secret, fmt.Errorf("failed to create jwt secret: %v", err) } - jwtPath := path.Join(os.TempDir(), "jwt_secret") + jwtPath := filepath.Join(os.TempDir(), "jwt_secret") if err := os.WriteFile(jwtPath, []byte(hexutil.Encode(secret[:])), 0600); err != nil { return "", secret, fmt.Errorf("failed to prepare jwt secret file: %v", err) } diff --git a/cmd/era/main.go b/cmd/era/main.go index c7f5de12bc1a..8b57fd695c53 100644 --- a/cmd/era/main.go +++ b/cmd/era/main.go @@ -22,7 +22,7 @@ import ( "fmt" "math/big" "os" - "path" + "path/filepath" "strconv" "strings" "time" @@ -176,7 +176,7 @@ func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { if epoch >= uint64(len(entries)) { return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch) } - return era.Open(path.Join(dir, entries[epoch])) + return era.Open(filepath.Join(dir, entries[epoch])) } // verify checks each era1 file in a directory to ensure it is well-formed and @@ -212,7 +212,7 @@ func verify(ctx *cli.Context) error { // Wrap in function so defers don't stack. err := func() error { name := entries[i] - e, err := era.Open(path.Join(dir, name)) + e, err := era.Open(filepath.Join(dir, name)) if err != nil { return fmt.Errorf("error opening era1 file %s: %w", name, err) } diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index aa0483a8ba67..a9489d069a70 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -22,7 +22,7 @@ import ( "fmt" "math/big" "os" - "path" + "path/filepath" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -96,7 +96,7 @@ func Transition(ctx *cli.Context) error { Debug: true, } getTracer = func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) { - traceFile, err := os.Create(path.Join(baseDir, fmt.Sprintf("trace-%d-%v.jsonl", txIndex, txHash.String()))) + traceFile, err := os.Create(filepath.Join(baseDir, fmt.Sprintf("trace-%d-%v.jsonl", txIndex, txHash.String()))) if err != nil { return nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err)) } @@ -108,7 +108,7 @@ func Transition(ctx *cli.Context) error { config = []byte(ctx.String(TraceTracerConfigFlag.Name)) } getTracer = func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) { - traceFile, err := os.Create(path.Join(baseDir, fmt.Sprintf("trace-%d-%v.json", txIndex, txHash.String()))) + traceFile, err := os.Create(filepath.Join(baseDir, fmt.Sprintf("trace-%d-%v.json", txIndex, txHash.String()))) if err != nil { return nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err)) } @@ -302,7 +302,7 @@ func saveFile(baseDir, filename string, data interface{}) error { if err != nil { return NewError(ErrorJson, fmt.Errorf("failed marshalling output: %v", err)) } - location := path.Join(baseDir, filename) + location := filepath.Join(baseDir, filename) if err = os.WriteFile(location, b, 0644); err != nil { return NewError(ErrorIO, fmt.Errorf("failed writing output: %v", err)) } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 37736dda8509..fc66e11dca96 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -27,7 +27,7 @@ import ( "io" "os" "os/signal" - "path" + "path/filepath" "runtime" "strings" "syscall" @@ -251,7 +251,7 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ if err != nil { return fmt.Errorf("error reading %s: %w", dir, err) } - checksums, err := readList(path.Join(dir, "checksums.txt")) + checksums, err := readList(filepath.Join(dir, "checksums.txt")) if err != nil { return fmt.Errorf("unable to read checksums.txt: %w", err) } @@ -268,7 +268,7 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ ) for i, filename := range entries { err := func() error { - f, err := os.Open(path.Join(dir, filename)) + f, err := os.Open(filepath.Join(dir, filename)) if err != nil { return fmt.Errorf("unable to open era: %w", err) } @@ -425,7 +425,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er ) for i := first; i <= last; i += step { err := func() error { - filename := path.Join(dir, era.Filename(network, int(i/step), common.Hash{})) + filename := filepath.Join(dir, era.Filename(network, int(i/step), common.Hash{})) f, err := os.Create(filename) if err != nil { return fmt.Errorf("could not create era file: %w", err) @@ -458,7 +458,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er return fmt.Errorf("export failed to finalize %d: %w", step/i, err) } // Set correct filename with root. - os.Rename(filename, path.Join(dir, era.Filename(network, int(i/step), root))) + os.Rename(filename, filepath.Join(dir, era.Filename(network, int(i/step), root))) // Compute checksum of entire Era1. if _, err := f.Seek(0, io.SeekStart); err != nil { @@ -481,7 +481,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er } } - os.WriteFile(path.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) + os.WriteFile(filepath.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) log.Info("Exported blockchain to", "dir", dir) diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index 9b7f1797d8dd..1d8e48344a2e 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -22,7 +22,7 @@ import ( "io" "math/big" "os" - "path" + "path/filepath" "strings" "testing" @@ -99,7 +99,7 @@ func TestHistoryImportAndExport(t *testing.T) { } // Read checksums. - b, err := os.ReadFile(path.Join(dir, "checksums.txt")) + b, err := os.ReadFile(filepath.Join(dir, "checksums.txt")) if err != nil { t.Fatalf("failed to read checksums: %v", err) } @@ -109,7 +109,7 @@ func TestHistoryImportAndExport(t *testing.T) { entries, _ := era.ReadDir(dir, "mainnet") for i, filename := range entries { func() { - f, err := os.Open(path.Join(dir, filename)) + f, err := os.Open(filepath.Join(dir, filename)) if err != nil { t.Fatalf("error opening era file: %v", err) } diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index b6a299f8ba89..a4761f337b85 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -22,7 +22,7 @@ package core import ( "math/big" - "path" + "path/filepath" "testing" "time" @@ -1762,7 +1762,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s // Create a temporary persistent database datadir := t.TempDir() - ancient := path.Join(datadir, "ancient") + ancient := filepath.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, @@ -1912,7 +1912,7 @@ func testIssue23496(t *testing.T, scheme string) { // Create a temporary persistent database datadir := t.TempDir() - ancient := path.Join(datadir, "ancient") + ancient := filepath.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index dd012c430c4d..80f8035df151 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -24,7 +24,7 @@ import ( "fmt" "math/big" "os" - "path" + "path/filepath" "strings" "testing" "time" @@ -63,7 +63,7 @@ type snapshotTestBasic struct { func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) { // Create a temporary persistent database datadir := t.TempDir() - ancient := path.Join(datadir, "ancient") + ancient := filepath.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index b4bd6a382a86..2a156638903d 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -24,6 +24,7 @@ import ( "math/rand" "os" "path" + "path/filepath" "sync" "testing" @@ -393,7 +394,7 @@ func TestRenameWindows(t *testing.T) { dir2 := t.TempDir() // Create file in dir1 and fill with data - f, err := os.Create(path.Join(dir1, fname)) + f, err := os.Create(filepath.Join(dir1, fname)) if err != nil { t.Fatal(err) } From 99bbbc0277e34fc3a31512a345ba20874ae98e18 Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Tue, 12 Mar 2024 19:12:37 +0800 Subject: [PATCH 81/91] internal/build, rpc: add missing HTTP response body Close() calls (#29223) Co-authored-by: Felix Lange --- internal/build/download.go | 6 ++++-- rpc/http.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/build/download.go b/internal/build/download.go index fda573df8331..206c51dce1e3 100644 --- a/internal/build/download.go +++ b/internal/build/download.go @@ -84,10 +84,12 @@ func (db *ChecksumDB) DownloadFile(url, dstPath string) error { resp, err := http.Get(url) if err != nil { return fmt.Errorf("download error: %v", err) - } else if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download error: status %d", resp.StatusCode) } defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download error: status %d", resp.StatusCode) + } if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { return err } diff --git a/rpc/http.go b/rpc/http.go index dd376b1ecd59..f4b99429ef4f 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -236,7 +236,7 @@ func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadClos if _, err := buf.ReadFrom(resp.Body); err == nil { body = buf.Bytes() } - + resp.Body.Close() return nil, HTTPError{ Status: resp.Status, StatusCode: resp.StatusCode, From 4bd55a064ccc804127de09397273d16966fe8a37 Mon Sep 17 00:00:00 2001 From: Aaron Chen Date: Tue, 12 Mar 2024 20:05:31 +0800 Subject: [PATCH 82/91] common/math: copy result in Exp (#29233) common/math: does not change base parameter --- common/math/big.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/common/math/big.go b/common/math/big.go index 013c0ba4b66f..721b297c9c4d 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -224,7 +224,7 @@ func ReadBits(bigint *big.Int, buf []byte) { } } -// U256 encodes as a 256 bit two's complement number. This operation is destructive. +// U256 encodes x as a 256 bit two's complement number. This operation is destructive. func U256(x *big.Int) *big.Int { return x.And(x, tt256m1) } @@ -255,14 +255,15 @@ func S256(x *big.Int) *big.Int { // // Courtesy @karalabe and @chfast func Exp(base, exponent *big.Int) *big.Int { + copyBase := new(big.Int).Set(base) result := big.NewInt(1) for _, word := range exponent.Bits() { for i := 0; i < wordBits; i++ { if word&1 == 1 { - U256(result.Mul(result, base)) + U256(result.Mul(result, copyBase)) } - U256(base.Mul(base, base)) + U256(copyBase.Mul(copyBase, copyBase)) word >>= 1 } } From 6c76b813df6d53b86fac17471e9a31afd20c481e Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 12 Mar 2024 14:29:35 +0100 Subject: [PATCH 83/91] miner: add additional log (#29193) Adds a debug level log if the payload building failed for whatever reason --- miner/payload_building.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/miner/payload_building.go b/miner/payload_building.go index cbdb82a642cf..d027cd1e1f3a 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -229,6 +229,8 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs) (*Payload, error) { r := miner.generateWork(fullParams) if r.err == nil { payload.update(r, time.Since(start)) + } else { + log.Info("Error while generating work", "id", payload.id, "err", r.err) } timer.Reset(miner.config.Recommit) case <-payload.stop: From 758fce71fab5289e3af711b1fa21a541c77cc435 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 12 Mar 2024 19:23:24 +0100 Subject: [PATCH 84/91] p2p: fix race in dialScheduler (#29235) Co-authored-by: Stefan --- p2p/dial.go | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/p2p/dial.go b/p2p/dial.go index 5e4ab1d50dcc..08e1db28771e 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -25,6 +25,7 @@ import ( mrand "math/rand" "net" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common/mclock" @@ -248,7 +249,7 @@ loop: } case task := <-d.doneCh: - id := task.dest.ID() + id := task.dest().ID() delete(d.dialing, id) d.updateStaticPool(id) d.doneSinceLastLog++ @@ -410,7 +411,7 @@ func (d *dialScheduler) startStaticDials(n int) (started int) { // updateStaticPool attempts to move the given static dial back into staticPool. func (d *dialScheduler) updateStaticPool(id enode.ID) { task, ok := d.static[id] - if ok && task.staticPoolIndex < 0 && d.checkDial(task.dest) == nil { + if ok && task.staticPoolIndex < 0 && d.checkDial(task.dest()) == nil { d.addToStaticPool(task) } } @@ -437,10 +438,11 @@ func (d *dialScheduler) removeFromStaticPool(idx int) { // startDial runs the given dial task in a separate goroutine. func (d *dialScheduler) startDial(task *dialTask) { - d.log.Trace("Starting p2p dial", "id", task.dest.ID(), "ip", task.dest.IP(), "flag", task.flags) - hkey := string(task.dest.ID().Bytes()) + node := task.dest() + d.log.Trace("Starting p2p dial", "id", node.ID(), "ip", node.IP(), "flag", task.flags) + hkey := string(node.ID().Bytes()) d.history.add(hkey, d.clock.Now().Add(dialHistoryExpiration)) - d.dialing[task.dest.ID()] = task + d.dialing[node.ID()] = task go func() { task.run(d) d.doneCh <- task @@ -451,39 +453,46 @@ func (d *dialScheduler) startDial(task *dialTask) { type dialTask struct { staticPoolIndex int flags connFlag + // These fields are private to the task and should not be // accessed by dialScheduler while the task is running. - dest *enode.Node + destPtr atomic.Pointer[enode.Node] lastResolved mclock.AbsTime resolveDelay time.Duration } func newDialTask(dest *enode.Node, flags connFlag) *dialTask { - return &dialTask{dest: dest, flags: flags, staticPoolIndex: -1} + t := &dialTask{flags: flags, staticPoolIndex: -1} + t.destPtr.Store(dest) + return t } type dialError struct { error } +func (t *dialTask) dest() *enode.Node { + return t.destPtr.Load() +} + func (t *dialTask) run(d *dialScheduler) { if t.needResolve() && !t.resolve(d) { return } - err := t.dial(d, t.dest) + err := t.dial(d, t.dest()) if err != nil { // For static nodes, resolve one more time if dialing fails. if _, ok := err.(*dialError); ok && t.flags&staticDialedConn != 0 { if t.resolve(d) { - t.dial(d, t.dest) + t.dial(d, t.dest()) } } } } func (t *dialTask) needResolve() bool { - return t.flags&staticDialedConn != 0 && t.dest.IP() == nil + return t.flags&staticDialedConn != 0 && t.dest().IP() == nil } // resolve attempts to find the current endpoint for the destination @@ -502,29 +511,31 @@ func (t *dialTask) resolve(d *dialScheduler) bool { if t.lastResolved > 0 && time.Duration(d.clock.Now()-t.lastResolved) < t.resolveDelay { return false } - resolved := d.resolver.Resolve(t.dest) + + node := t.dest() + resolved := d.resolver.Resolve(node) t.lastResolved = d.clock.Now() if resolved == nil { t.resolveDelay *= 2 if t.resolveDelay > maxResolveDelay { t.resolveDelay = maxResolveDelay } - d.log.Debug("Resolving node failed", "id", t.dest.ID(), "newdelay", t.resolveDelay) + d.log.Debug("Resolving node failed", "id", node.ID(), "newdelay", t.resolveDelay) return false } // The node was found. t.resolveDelay = initialResolveDelay - t.dest = resolved - d.log.Debug("Resolved node", "id", t.dest.ID(), "addr", &net.TCPAddr{IP: t.dest.IP(), Port: t.dest.TCP()}) + t.destPtr.Store(resolved) + d.log.Debug("Resolved node", "id", resolved.ID(), "addr", &net.TCPAddr{IP: resolved.IP(), Port: resolved.TCP()}) return true } // dial performs the actual connection attempt. func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { dialMeter.Mark(1) - fd, err := d.dialer.Dial(d.ctx, t.dest) + fd, err := d.dialer.Dial(d.ctx, dest) if err != nil { - d.log.Trace("Dial error", "id", t.dest.ID(), "addr", nodeAddr(t.dest), "conn", t.flags, "err", cleanupDialErr(err)) + d.log.Trace("Dial error", "id", dest.ID(), "addr", nodeAddr(dest), "conn", t.flags, "err", cleanupDialErr(err)) dialConnectionError.Mark(1) return &dialError{err} } @@ -532,8 +543,9 @@ func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { } func (t *dialTask) String() string { - id := t.dest.ID() - return fmt.Sprintf("%v %x %v:%d", t.flags, id[:8], t.dest.IP(), t.dest.TCP()) + node := t.dest() + id := node.ID() + return fmt.Sprintf("%v %x %v:%d", t.flags, id[:8], node.IP(), node.TCP()) } func cleanupDialErr(err error) error { From eff424cc302152f3914e3f9c8b49efe92e33353f Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Wed, 13 Mar 2024 07:40:02 +0100 Subject: [PATCH 85/91] eth/tracers: fix concurrency issue for JS-tracing a block (#29238) This change fixes a concurrency-issue where JS-tracers were accessing the block-ctx GetHash function in a in parallel, which is not safe. --- eth/tracers/api.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index fa8c881d1a71..0add06c8f69b 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -632,7 +632,6 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat var ( txs = block.Transactions() blockHash = block.Hash() - blockCtx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) signer = types.MakeSigner(api.backend.ChainConfig(), block.Number(), block.Time()) results = make([]*txTraceResult, len(txs)) pend sync.WaitGroup @@ -655,6 +654,11 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat TxIndex: task.index, TxHash: txs[task.index].Hash(), } + // Reconstruct the block context for each transaction + // as the GetHash function of BlockContext is not safe for + // concurrent use. + // See: https://github.com/ethereum/go-ethereum/issues/29114 + blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) res, err := api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config) if err != nil { results[task.index] = &txTraceResult{TxHash: txs[task.index].Hash(), Error: err.Error()} @@ -667,6 +671,7 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat // Feed the transactions into the tracers and return var failed error + blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) txloop: for i, tx := range txs { // Send the trace task over for execution From d5bacfa4def558a4c7b261c1a9fbfdbfc295e491 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Wed, 13 Mar 2024 07:51:46 +0100 Subject: [PATCH 86/91] crypto/kz4844: pass blobs by ref (#29050) This change makes use of the following underlying changes to the kzg-libraries in order to avoid passing large things on the stack: - c-kzg: https://github.com/ethereum/c-kzg-4844/pull/393 and - go-kzg: https://github.com/crate-crypto/go-kzg-4844/pull/63 --- cmd/devp2p/internal/ethtest/suite.go | 4 ++-- core/txpool/blobpool/blobpool_test.go | 4 ++-- core/txpool/validation.go | 2 +- core/types/tx_blob_test.go | 4 ++-- crypto/kzg4844/kzg4844.go | 8 ++++---- crypto/kzg4844/kzg4844_ckzg_cgo.go | 16 ++++++++-------- crypto/kzg4844/kzg4844_ckzg_nocgo.go | 8 ++++---- crypto/kzg4844/kzg4844_gokzg.go | 16 ++++++++-------- crypto/kzg4844/kzg4844_test.go | 4 ++-- go.mod | 4 ++-- go.sum | 8 ++++---- internal/ethapi/api_test.go | 17 +++++++++-------- internal/ethapi/transaction_args.go | 6 +++--- 13 files changed, 51 insertions(+), 50 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index d9efe2624432..b5cc27a2b590 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -754,8 +754,8 @@ func makeSidecar(data ...byte) *types.BlobTxSidecar { ) for i := range blobs { blobs[i][0] = data[i] - c, _ := kzg4844.BlobToCommitment(blobs[i]) - p, _ := kzg4844.ComputeBlobProof(blobs[i], c) + c, _ := kzg4844.BlobToCommitment(&blobs[i]) + p, _ := kzg4844.ComputeBlobProof(&blobs[i], c) commitments = append(commitments, c) proofs = append(proofs, p) } diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index bac239db4769..279750c73f2a 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -48,7 +48,7 @@ import ( ) var ( - emptyBlob = kzg4844.Blob{} + emptyBlob = new(kzg4844.Blob) emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) emptyBlobVHash = kzg4844.CalcBlobHashV1(sha256.New(), &emptyBlobCommit) @@ -198,7 +198,7 @@ func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap BlobHashes: []common.Hash{emptyBlobVHash}, Value: uint256.NewInt(100), Sidecar: &types.BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: []kzg4844.Blob{*emptyBlob}, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 63f127f55ca2..d9a85a435da4 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -162,7 +162,7 @@ func validateBlobSidecar(hashes []common.Hash, sidecar *types.BlobTxSidecar) err // Blob commitments match with the hashes in the transaction, verify the // blobs themselves via KZG for i := range sidecar.Blobs { - if err := kzg4844.VerifyBlobProof(sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil { + if err := kzg4844.VerifyBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil { return fmt.Errorf("invalid blob %d: %v", i, err) } } diff --git a/core/types/tx_blob_test.go b/core/types/tx_blob_test.go index 25d09e31ce4a..6bd0f183b730 100644 --- a/core/types/tx_blob_test.go +++ b/core/types/tx_blob_test.go @@ -59,7 +59,7 @@ func TestBlobTxSize(t *testing.T) { } var ( - emptyBlob = kzg4844.Blob{} + emptyBlob = new(kzg4844.Blob) emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) ) @@ -72,7 +72,7 @@ func createEmptyBlobTx(key *ecdsa.PrivateKey, withSidecar bool) *Transaction { func createEmptyBlobTxInner(withSidecar bool) *BlobTx { sidecar := &BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: []kzg4844.Blob{*emptyBlob}, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, } diff --git a/crypto/kzg4844/kzg4844.go b/crypto/kzg4844/kzg4844.go index 168ff8347095..39fdfbe740ee 100644 --- a/crypto/kzg4844/kzg4844.go +++ b/crypto/kzg4844/kzg4844.go @@ -105,7 +105,7 @@ func UseCKZG(use bool) error { } // BlobToCommitment creates a small commitment out of a data blob. -func BlobToCommitment(blob Blob) (Commitment, error) { +func BlobToCommitment(blob *Blob) (Commitment, error) { if useCKZG.Load() { return ckzgBlobToCommitment(blob) } @@ -114,7 +114,7 @@ func BlobToCommitment(blob Blob) (Commitment, error) { // ComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func ComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func ComputeProof(blob *Blob, point Point) (Proof, Claim, error) { if useCKZG.Load() { return ckzgComputeProof(blob, point) } @@ -134,7 +134,7 @@ func VerifyProof(commitment Commitment, point Point, claim Claim, proof Proof) e // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func ComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func ComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { if useCKZG.Load() { return ckzgComputeBlobProof(blob, commitment) } @@ -142,7 +142,7 @@ func ComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { } // VerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func VerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func VerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { if useCKZG.Load() { return ckzgVerifyBlobProof(blob, commitment, proof) } diff --git a/crypto/kzg4844/kzg4844_ckzg_cgo.go b/crypto/kzg4844/kzg4844_ckzg_cgo.go index 54002856987c..11bc451b58a9 100644 --- a/crypto/kzg4844/kzg4844_ckzg_cgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_cgo.go @@ -61,10 +61,10 @@ func ckzgInit() { } // ckzgBlobToCommitment creates a small commitment out of a data blob. -func ckzgBlobToCommitment(blob Blob) (Commitment, error) { +func ckzgBlobToCommitment(blob *Blob) (Commitment, error) { ckzgIniter.Do(ckzgInit) - commitment, err := ckzg4844.BlobToKZGCommitment((ckzg4844.Blob)(blob)) + commitment, err := ckzg4844.BlobToKZGCommitment((*ckzg4844.Blob)(blob)) if err != nil { return Commitment{}, err } @@ -73,10 +73,10 @@ func ckzgBlobToCommitment(blob Blob) (Commitment, error) { // ckzgComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func ckzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func ckzgComputeProof(blob *Blob, point Point) (Proof, Claim, error) { ckzgIniter.Do(ckzgInit) - proof, claim, err := ckzg4844.ComputeKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes32)(point)) + proof, claim, err := ckzg4844.ComputeKZGProof((*ckzg4844.Blob)(blob), (ckzg4844.Bytes32)(point)) if err != nil { return Proof{}, Claim{}, err } @@ -102,10 +102,10 @@ func ckzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Proo // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func ckzgComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { ckzgIniter.Do(ckzgInit) - proof, err := ckzg4844.ComputeBlobKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment)) + proof, err := ckzg4844.ComputeBlobKZGProof((*ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment)) if err != nil { return Proof{}, err } @@ -113,10 +113,10 @@ func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { } // ckzgVerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func ckzgVerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func ckzgVerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { ckzgIniter.Do(ckzgInit) - valid, err := ckzg4844.VerifyBlobKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment), (ckzg4844.Bytes48)(proof)) + valid, err := ckzg4844.VerifyBlobKZGProof((*ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment), (ckzg4844.Bytes48)(proof)) if err != nil { return err } diff --git a/crypto/kzg4844/kzg4844_ckzg_nocgo.go b/crypto/kzg4844/kzg4844_ckzg_nocgo.go index ed840c75bb68..70a78e80d16a 100644 --- a/crypto/kzg4844/kzg4844_ckzg_nocgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_nocgo.go @@ -32,13 +32,13 @@ func ckzgInit() { } // ckzgBlobToCommitment creates a small commitment out of a data blob. -func ckzgBlobToCommitment(blob Blob) (Commitment, error) { +func ckzgBlobToCommitment(blob *Blob) (Commitment, error) { panic("unsupported platform") } // ckzgComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func ckzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func ckzgComputeProof(blob *Blob, point Point) (Proof, Claim, error) { panic("unsupported platform") } @@ -52,11 +52,11 @@ func ckzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Proo // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func ckzgComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { panic("unsupported platform") } // ckzgVerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func ckzgVerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func ckzgVerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { panic("unsupported platform") } diff --git a/crypto/kzg4844/kzg4844_gokzg.go b/crypto/kzg4844/kzg4844_gokzg.go index 3f03bb52738e..b4af9b1671e9 100644 --- a/crypto/kzg4844/kzg4844_gokzg.go +++ b/crypto/kzg4844/kzg4844_gokzg.go @@ -46,10 +46,10 @@ func gokzgInit() { } // gokzgBlobToCommitment creates a small commitment out of a data blob. -func gokzgBlobToCommitment(blob Blob) (Commitment, error) { +func gokzgBlobToCommitment(blob *Blob) (Commitment, error) { gokzgIniter.Do(gokzgInit) - commitment, err := context.BlobToKZGCommitment((gokzg4844.Blob)(blob), 0) + commitment, err := context.BlobToKZGCommitment((*gokzg4844.Blob)(blob), 0) if err != nil { return Commitment{}, err } @@ -58,10 +58,10 @@ func gokzgBlobToCommitment(blob Blob) (Commitment, error) { // gokzgComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func gokzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func gokzgComputeProof(blob *Blob, point Point) (Proof, Claim, error) { gokzgIniter.Do(gokzgInit) - proof, claim, err := context.ComputeKZGProof((gokzg4844.Blob)(blob), (gokzg4844.Scalar)(point), 0) + proof, claim, err := context.ComputeKZGProof((*gokzg4844.Blob)(blob), (gokzg4844.Scalar)(point), 0) if err != nil { return Proof{}, Claim{}, err } @@ -80,10 +80,10 @@ func gokzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Pro // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func gokzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func gokzgComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { gokzgIniter.Do(gokzgInit) - proof, err := context.ComputeBlobKZGProof((gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), 0) + proof, err := context.ComputeBlobKZGProof((*gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), 0) if err != nil { return Proof{}, err } @@ -91,8 +91,8 @@ func gokzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { } // gokzgVerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func gokzgVerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func gokzgVerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { gokzgIniter.Do(gokzgInit) - return context.VerifyBlobKZGProof((gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), (gokzg4844.KZGProof)(proof)) + return context.VerifyBlobKZGProof((*gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), (gokzg4844.KZGProof)(proof)) } diff --git a/crypto/kzg4844/kzg4844_test.go b/crypto/kzg4844/kzg4844_test.go index fae8a7a76eaf..a6782d4768ad 100644 --- a/crypto/kzg4844/kzg4844_test.go +++ b/crypto/kzg4844/kzg4844_test.go @@ -36,13 +36,13 @@ func randFieldElement() [32]byte { return gokzg4844.SerializeScalar(r) } -func randBlob() Blob { +func randBlob() *Blob { var blob Blob for i := 0; i < len(blob); i += gokzg4844.SerializedScalarSize { fieldElementBytes := randFieldElement() copy(blob[i:i+gokzg4844.SerializedScalarSize], fieldElementBytes[:]) } - return blob + return &blob } func TestCKZGWithPoint(t *testing.T) { testKZGWithPoint(t, true) } diff --git a/go.mod b/go.mod index ca45364b8bf2..1e0344594ac3 100644 --- a/go.mod +++ b/go.mod @@ -16,12 +16,12 @@ require ( github.com/cockroachdb/pebble v1.1.0 github.com/consensys/gnark-crypto v0.12.1 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 - github.com/crate-crypto/go-kzg-4844 v0.7.0 + github.com/crate-crypto/go-kzg-4844 v1.0.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 - github.com/ethereum/c-kzg-4844 v0.4.0 + github.com/ethereum/c-kzg-4844 v1.0.0 github.com/fatih/color v1.13.0 github.com/ferranbt/fastssz v0.1.2 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e diff --git a/go.sum b/go.sum index 18236bf8e74d..98137d3e442e 100644 --- a/go.sum +++ b/go.sum @@ -129,8 +129,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= -github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= -github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -160,8 +160,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= -github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk= diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 3f69f861444c..5636309589df 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -1088,7 +1088,8 @@ func TestFillBlobTransaction(t *testing.T) { Config: params.MergedTestChainConfig, Alloc: types.GenesisAlloc{}, } - emptyBlob = kzg4844.Blob{} + emptyBlob = new(kzg4844.Blob) + emptyBlobs = []kzg4844.Blob{*emptyBlob} emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) emptyBlobHash common.Hash = kzg4844.CalcBlobHashV1(sha256.New(), &emptyBlobCommit) @@ -1171,14 +1172,14 @@ func TestFillBlobTransaction(t *testing.T) { From: &b.acc.Address, To: &to, Value: (*hexutil.Big)(big.NewInt(1)), - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, want: &result{ Hashes: []common.Hash{emptyBlobHash}, Sidecar: &types.BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, @@ -1191,14 +1192,14 @@ func TestFillBlobTransaction(t *testing.T) { To: &to, Value: (*hexutil.Big)(big.NewInt(1)), BlobHashes: []common.Hash{emptyBlobHash}, - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, want: &result{ Hashes: []common.Hash{emptyBlobHash}, Sidecar: &types.BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, @@ -1211,7 +1212,7 @@ func TestFillBlobTransaction(t *testing.T) { To: &to, Value: (*hexutil.Big)(big.NewInt(1)), BlobHashes: []common.Hash{{0x01, 0x22}}, - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, @@ -1223,12 +1224,12 @@ func TestFillBlobTransaction(t *testing.T) { From: &b.acc.Address, To: &to, Value: (*hexutil.Big)(big.NewInt(1)), - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, }, want: &result{ Hashes: []common.Hash{emptyBlobHash}, Sidecar: &types.BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index bae1c6864159..2751d5b5aae6 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -326,12 +326,12 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, b Backend) er commitments := make([]kzg4844.Commitment, n) proofs := make([]kzg4844.Proof, n) for i, b := range args.Blobs { - c, err := kzg4844.BlobToCommitment(b) + c, err := kzg4844.BlobToCommitment(&b) if err != nil { return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) } commitments[i] = c - p, err := kzg4844.ComputeBlobProof(b, c) + p, err := kzg4844.ComputeBlobProof(&b, c) if err != nil { return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) } @@ -341,7 +341,7 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, b Backend) er args.Proofs = proofs } else { for i, b := range args.Blobs { - if err := kzg4844.VerifyBlobProof(b, args.Commitments[i], args.Proofs[i]); err != nil { + if err := kzg4844.VerifyBlobProof(&b, args.Commitments[i], args.Proofs[i]); err != nil { return fmt.Errorf("failed to verify blob proof: %v", err) } } From b80643b7370075262fd6dfad7ae8aa77710e2ef1 Mon Sep 17 00:00:00 2001 From: Justin Dhillon Date: Tue, 12 Mar 2024 23:54:40 -0700 Subject: [PATCH 87/91] accounts/usbwallet, common/bitutil: fix broken links in docs (#29078) fixes some links in documentation --- accounts/usbwallet/ledger.go | 2 +- common/bitutil/bitutil.go | 2 +- common/bitutil/bitutil_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index d0cb93e74e00..81836b3717ac 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -16,7 +16,7 @@ // This file contains the implementation for interacting with the Ledger hardware // wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo: -// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc +// https://github.com/LedgerHQ/app-ethereum/blob/develop/doc/ethapp.adoc package usbwallet diff --git a/common/bitutil/bitutil.go b/common/bitutil/bitutil.go index cd3e72169fc5..a18a6d18eed8 100644 --- a/common/bitutil/bitutil.go +++ b/common/bitutil/bitutil.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Adapted from: https://golang.org/src/crypto/cipher/xor.go +// Adapted from: https://go.dev/src/crypto/subtle/xor_generic.go // Package bitutil implements fast bitwise operations. package bitutil diff --git a/common/bitutil/bitutil_test.go b/common/bitutil/bitutil_test.go index 307bf731f765..12f3fe24a6c9 100644 --- a/common/bitutil/bitutil_test.go +++ b/common/bitutil/bitutil_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Adapted from: https://golang.org/src/crypto/cipher/xor_test.go +// Adapted from: https://go.dev/src/crypto/subtle/xor_test.go package bitutil From c170fa277cbf2a9faf9f35665f1ba8f34f94062a Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 13 Mar 2024 19:39:30 +0800 Subject: [PATCH 88/91] core: improve chain rewinding mechanism (#29196) * core: improve chain rewinding mechanism * core: address comment * core: periodically print progress log * core: address comments * core: fix comment * core: fix rewinding in path * core: fix beyondRoot condition * core: polish code * core: polish code * core: extend code comment * core: stop rewinding if chain is gapped or genesis is reached * core: fix broken tests --- core/blockchain.go | 238 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 183 insertions(+), 55 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 9bd7fdcd9544..ba346b010d47 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -616,6 +616,172 @@ func (bc *BlockChain) SetSafe(header *types.Header) { } } +// rewindPathHead implements the logic of rewindHead in the context of hash scheme. +func (bc *BlockChain) rewindHashHead(head *types.Header, root common.Hash) (*types.Header, uint64) { + var ( + limit uint64 // The oldest block that will be searched for this rewinding + beyondRoot = root == common.Hash{} // Flag whether we're beyond the requested root (no root, always true) + pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot point state + rootNumber uint64 // Associated block number of requested root + + start = time.Now() // Timestamp the rewinding is restarted + logged = time.Now() // Timestamp last progress log was printed + ) + // The oldest block to be searched is determined by the pivot block or a constant + // searching threshold. The rationale behind this is as follows: + // + // - Snap sync is selected if the pivot block is available. The earliest available + // state is the pivot block itself, so there is no sense in going further back. + // + // - Full sync is selected if the pivot block does not exist. The hash database + // periodically flushes the state to disk, and the used searching threshold is + // considered sufficient to find a persistent state, even for the testnet. It + // might be not enough for a chain that is nearly empty. In the worst case, + // the entire chain is reset to genesis, and snap sync is re-enabled on top, + // which is still acceptable. + if pivot != nil { + limit = *pivot + } else if head.Number.Uint64() > params.FullImmutabilityThreshold { + limit = head.Number.Uint64() - params.FullImmutabilityThreshold + } + for { + logger := log.Trace + if time.Since(logged) > time.Second*8 { + logged = time.Now() + logger = log.Info + } + logger("Block state missing, rewinding further", "number", head.Number, "hash", head.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) + + // If a root threshold was requested but not yet crossed, check + if !beyondRoot && head.Root == root { + beyondRoot, rootNumber = true, head.Number.Uint64() + } + // If search limit is reached, return the genesis block as the + // new chain head. + if head.Number.Uint64() < limit { + log.Info("Rewinding limit reached, resetting to genesis", "number", head.Number, "hash", head.Hash(), "limit", limit) + return bc.genesisBlock.Header(), rootNumber + } + // If the associated state is not reachable, continue searching + // backwards until an available state is found. + if !bc.HasState(head.Root) { + // If the chain is gapped in the middle, return the genesis + // block as the new chain head. + parent := bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) + if parent == nil { + log.Error("Missing block in the middle, resetting to genesis", "number", head.Number.Uint64()-1, "hash", head.ParentHash) + return bc.genesisBlock.Header(), rootNumber + } + head = parent + + // If the genesis block is reached, stop searching. + if head.Number.Uint64() == 0 { + log.Info("Genesis block reached", "number", head.Number, "hash", head.Hash()) + return head, rootNumber + } + continue // keep rewinding + } + // Once the available state is found, ensure that the requested root + // has already been crossed. If not, continue rewinding. + if beyondRoot || head.Number.Uint64() == 0 { + log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) + return head, rootNumber + } + log.Debug("Skipping block with threshold state", "number", head.Number, "hash", head.Hash(), "root", head.Root) + head = bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) // Keep rewinding + } +} + +// rewindPathHead implements the logic of rewindHead in the context of path scheme. +func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*types.Header, uint64) { + var ( + pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot block + rootNumber uint64 // Associated block number of requested root + + // BeyondRoot represents whether the requested root is already + // crossed. The flag value is set to true if the root is empty. + beyondRoot = root == common.Hash{} + + // noState represents if the target state requested for search + // is unavailable and impossible to be recovered. + noState = !bc.HasState(root) && !bc.stateRecoverable(root) + + start = time.Now() // Timestamp the rewinding is restarted + logged = time.Now() // Timestamp last progress log was printed + ) + // Rewind the head block tag until an available state is found. + for { + logger := log.Trace + if time.Since(logged) > time.Second*8 { + logged = time.Now() + logger = log.Info + } + logger("Block state missing, rewinding further", "number", head.Number, "hash", head.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) + + // If a root threshold was requested but not yet crossed, check + if !beyondRoot && head.Root == root { + beyondRoot, rootNumber = true, head.Number.Uint64() + } + // If the root threshold hasn't been crossed but the available + // state is reached, quickly determine if the target state is + // possible to be reached or not. + if !beyondRoot && noState && bc.HasState(head.Root) { + beyondRoot = true + log.Info("Disable the search for unattainable state", "root", root) + } + // Check if the associated state is available or recoverable if + // the requested root has already been crossed. + if beyondRoot && (bc.HasState(head.Root) || bc.stateRecoverable(head.Root)) { + break + } + // If pivot block is reached, return the genesis block as the + // new chain head. Theoretically there must be a persistent + // state before or at the pivot block, prevent endless rewinding + // towards the genesis just in case. + if pivot != nil && *pivot >= head.Number.Uint64() { + log.Info("Pivot block reached, resetting to genesis", "number", head.Number, "hash", head.Hash()) + return bc.genesisBlock.Header(), rootNumber + } + // If the chain is gapped in the middle, return the genesis + // block as the new chain head + parent := bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) // Keep rewinding + if parent == nil { + log.Error("Missing block in the middle, resetting to genesis", "number", head.Number.Uint64()-1, "hash", head.ParentHash) + return bc.genesisBlock.Header(), rootNumber + } + head = parent + + // If the genesis block is reached, stop searching. + if head.Number.Uint64() == 0 { + log.Info("Genesis block reached", "number", head.Number, "hash", head.Hash()) + return head, rootNumber + } + } + // Recover if the target state if it's not available yet. + if !bc.HasState(head.Root) { + if err := bc.triedb.Recover(head.Root); err != nil { + log.Crit("Failed to rollback state", "err", err) + } + } + log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) + return head, rootNumber +} + +// rewindHead searches the available states in the database and returns the associated +// block as the new head block. +// +// If the given root is not empty, then the rewind should attempt to pass the specified +// state root and return the associated block number as well. If the root, typically +// representing the state corresponding to snapshot disk layer, is deemed impassable, +// then block number zero is returned, indicating that snapshot recovery is disabled +// and the whole snapshot should be auto-generated in case of head mismatch. +func (bc *BlockChain) rewindHead(head *types.Header, root common.Hash) (*types.Header, uint64) { + if bc.triedb.Scheme() == rawdb.PathScheme { + return bc.rewindPathHead(head, root) + } + return bc.rewindHashHead(head, root) +} + // setHeadBeyondRoot rewinds the local chain to a new head with the extra condition // that the rewind must pass the specified state root. This method is meant to be // used when rewinding with snapshots enabled to ensure that we go back further than @@ -634,79 +800,40 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha } defer bc.chainmu.Unlock() - // Track the block number of the requested root hash - var rootNumber uint64 // (no root == always 0) - - // Retrieve the last pivot block to short circuit rollbacks beyond it and the - // current freezer limit to start nuking id underflown - pivot := rawdb.ReadLastPivotNumber(bc.db) - frozen, _ := bc.db.Ancients() + var ( + // Track the block number of the requested root hash + rootNumber uint64 // (no root == always 0) + // Retrieve the last pivot block to short circuit rollbacks beyond it + // and the current freezer limit to start nuking it's underflown. + pivot = rawdb.ReadLastPivotNumber(bc.db) + ) updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (*types.Header, bool) { // Rewind the blockchain, ensuring we don't end up with a stateless head // block. Note, depth equality is permitted to allow using SetHead as a // chain reparation mechanism without deleting any data! if currentBlock := bc.CurrentBlock(); currentBlock != nil && header.Number.Uint64() <= currentBlock.Number.Uint64() { - newHeadBlock := bc.GetBlock(header.Hash(), header.Number.Uint64()) - if newHeadBlock == nil { - log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash()) - newHeadBlock = bc.genesisBlock - } else { - // Block exists. Keep rewinding until either we find one with state - // or until we exceed the optional threshold root hash - beyondRoot := (root == common.Hash{}) // Flag whether we're beyond the requested root (no root, always true) - - for { - // If a root threshold was requested but not yet crossed, check - if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root { - beyondRoot, rootNumber = true, newHeadBlock.NumberU64() - } - if !bc.HasState(newHeadBlock.Root()) && !bc.stateRecoverable(newHeadBlock.Root()) { - log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) - if pivot == nil || newHeadBlock.NumberU64() > *pivot { - parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) - if parent != nil { - newHeadBlock = parent - continue - } - log.Error("Missing block in the middle, aiming genesis", "number", newHeadBlock.NumberU64()-1, "hash", newHeadBlock.ParentHash()) - newHeadBlock = bc.genesisBlock - } else { - log.Trace("Rewind passed pivot, aiming genesis", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "pivot", *pivot) - newHeadBlock = bc.genesisBlock - } - } - if beyondRoot || newHeadBlock.NumberU64() == 0 { - if !bc.HasState(newHeadBlock.Root()) && bc.stateRecoverable(newHeadBlock.Root()) { - // Rewind to a block with recoverable state. If the state is - // missing, run the state recovery here. - if err := bc.triedb.Recover(newHeadBlock.Root()); err != nil { - log.Crit("Failed to rollback state", "err", err) // Shouldn't happen - } - log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) - } - break - } - log.Debug("Skipping block with threshold state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "root", newHeadBlock.Root()) - newHeadBlock = bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) // Keep rewinding - } - } + var newHeadBlock *types.Header + newHeadBlock, rootNumber = bc.rewindHead(header, root) rawdb.WriteHeadBlockHash(db, newHeadBlock.Hash()) // Degrade the chain markers if they are explicitly reverted. // In theory we should update all in-memory markers in the // last step, however the direction of SetHead is from high // to low, so it's safe to update in-memory markers directly. - bc.currentBlock.Store(newHeadBlock.Header()) - headBlockGauge.Update(int64(newHeadBlock.NumberU64())) + bc.currentBlock.Store(newHeadBlock) + headBlockGauge.Update(int64(newHeadBlock.Number.Uint64())) // The head state is missing, which is only possible in the path-based // scheme. This situation occurs when the chain head is rewound below // the pivot point. In this scenario, there is no possible recovery // approach except for rerunning a snap sync. Do nothing here until the // state syncer picks it up. - if !bc.HasState(newHeadBlock.Root()) { - log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number(), "hash", newHeadBlock.Hash()) + if !bc.HasState(newHeadBlock.Root) { + if newHeadBlock.Number.Uint64() != 0 { + log.Crit("Chain is stateless at a non-genesis block") + } + log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number, "hash", newHeadBlock.Hash()) } } // Rewind the snap block in a simpleton way to the target head @@ -733,6 +860,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha // intent afterwards is full block importing, delete the chain segment // between the stateful-block and the sethead target. var wipe bool + frozen, _ := bc.db.Ancients() if headNumber+1 < frozen { wipe = pivot == nil || headNumber >= *pivot } From f3d18d64bf4c026740ee6c8ae8949a8c19391b49 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Wed, 13 Mar 2024 18:12:23 +0100 Subject: [PATCH 89/91] tests, appveyor: only execute one in four permutations on CI (#29220) tests, appveyor: only execute one in four permutations when flag -short is used Also enable -short flag on all appveyor builds (also ubuntu) --- appveyor.yml | 2 +- tests/block_test.go | 41 +++++++++++++++++++++++++---------------- tests/state_test.go | 23 ++++++++++++++++++----- 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 4a8c4b737a2f..41c70491b4eb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,7 +26,7 @@ for: - go run build/ci.go lint - go run build/ci.go install -dlgo test_script: - - go run build/ci.go test -dlgo + - go run build/ci.go test -dlgo -short # linux/386 is disabled. - matrix: diff --git a/tests/block_test.go b/tests/block_test.go index fb355085fd8c..1ba84f5f24b6 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -18,7 +18,6 @@ package tests import ( "math/rand" - "runtime" "testing" "github.com/ethereum/go-ethereum/common" @@ -51,9 +50,6 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`.*randomStatetest94.json.*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { - if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 { - t.Skip("test (randomly) skipped on 32-bit windows") - } execBlockTest(t, bt, test) }) // There is also a LegacyTests folder, containing blockchain tests generated @@ -74,20 +70,33 @@ func TestExecutionSpecBlocktests(t *testing.T) { } func execBlockTest(t *testing.T, bt *testMatcher, test *BlockTest) { - if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil, nil)); err != nil { - t.Errorf("test in hash mode without snapshotter failed: %v", err) - return + // If -short flag is used, we don't execute all four permutations, only one. + executionMask := 0xf + if testing.Short() { + executionMask = (1 << (rand.Int63() & 4)) + } + if executionMask&0x1 != 0 { + if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil, nil)); err != nil { + t.Errorf("test in hash mode without snapshotter failed: %v", err) + return + } } - if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil, nil)); err != nil { - t.Errorf("test in hash mode with snapshotter failed: %v", err) - return + if executionMask&0x2 != 0 { + if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil, nil)); err != nil { + t.Errorf("test in hash mode with snapshotter failed: %v", err) + return + } } - if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil, nil)); err != nil { - t.Errorf("test in path mode without snapshotter failed: %v", err) - return + if executionMask&0x4 != 0 { + if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil, nil)); err != nil { + t.Errorf("test in path mode without snapshotter failed: %v", err) + return + } } - if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil, nil)); err != nil { - t.Errorf("test in path mode with snapshotter failed: %v", err) - return + if executionMask&0x8 != 0 { + if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil, nil)); err != nil { + t.Errorf("test in path mode with snapshotter failed: %v", err) + return + } } } diff --git a/tests/state_test.go b/tests/state_test.go index 1d749d8bcf52..6ec5c9d857bc 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -25,7 +25,6 @@ import ( "os" "path/filepath" "reflect" - "runtime" "strings" "testing" "time" @@ -99,15 +98,20 @@ func TestExecutionSpecState(t *testing.T) { } func execStateTest(t *testing.T, st *testMatcher, test *StateTest) { - if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 { - t.Skip("test (randomly) skipped on 32-bit windows") - return - } for _, subtest := range test.Subtests() { subtest := subtest key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) + // If -short flag is used, we don't execute all four permutations, only + // one. + executionMask := 0xf + if testing.Short() { + executionMask = (1 << (rand.Int63() & 4)) + } t.Run(key+"/hash/trie", func(t *testing.T) { + if executionMask&0x1 == 0 { + t.Skip("test (randomly) skipped due to short-tag") + } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { var result error test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, state *StateTestState) { @@ -117,6 +121,9 @@ func execStateTest(t *testing.T, st *testMatcher, test *StateTest) { }) }) t.Run(key+"/hash/snap", func(t *testing.T) { + if executionMask&0x2 == 0 { + t.Skip("test (randomly) skipped due to short-tag") + } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { var result error test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, state *StateTestState) { @@ -132,6 +139,9 @@ func execStateTest(t *testing.T, st *testMatcher, test *StateTest) { }) }) t.Run(key+"/path/trie", func(t *testing.T) { + if executionMask&0x4 == 0 { + t.Skip("test (randomly) skipped due to short-tag") + } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { var result error test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, state *StateTestState) { @@ -141,6 +151,9 @@ func execStateTest(t *testing.T, st *testMatcher, test *StateTest) { }) }) t.Run(key+"/path/snap", func(t *testing.T) { + if executionMask&0x8 == 0 { + t.Skip("test (randomly) skipped due to short-tag") + } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { var result error test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, state *StateTestState) { From 57308beecf7040391aee6c3102587063501f6825 Mon Sep 17 00:00:00 2001 From: Bin <49082129+songzhibin97@users.noreply.github.com> Date: Thu, 14 Mar 2024 07:25:42 +0800 Subject: [PATCH 90/91] go.mod: update golang.org/x/crypto from v0.17.0 to v0.21.0 (#29228) --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 1e0344594ac3..4e481065469f 100644 --- a/go.mod +++ b/go.mod @@ -66,10 +66,10 @@ require ( github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.25.7 go.uber.org/automaxprocs v1.5.2 - golang.org/x/crypto v0.17.0 + golang.org/x/crypto v0.21.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/sync v0.5.0 - golang.org/x/sys v0.16.0 + golang.org/x/sys v0.18.0 golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.15.0 @@ -141,7 +141,7 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.18.0 // indirect + golang.org/x/net v0.21.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 98137d3e442e..b5cb268a0bda 100644 --- a/go.sum +++ b/go.sum @@ -526,8 +526,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -599,8 +599,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -682,8 +682,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 3c26ffeb2968907f68d41faab757dacdcb280941 Mon Sep 17 00:00:00 2001 From: Haotian <51777534+tmelhao@users.noreply.github.com> Date: Thu, 14 Mar 2024 07:26:46 +0800 Subject: [PATCH 91/91] eth/catalyst: remove error return in delayPayloadImport (#29043) Co-authored-by: tmelhao --- eth/catalyst/api.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index e5781b2c8f3e..f549f29dc62b 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -567,7 +567,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe // update after legit payload executions. parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return api.delayPayloadImport(block) + return api.delayPayloadImport(block), nil } // We have an existing parent, do some sanity checks to avoid the beacon client // triggering too early @@ -593,7 +593,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe // into the database directly will conflict with the assumptions of snap sync // that it has an empty db that it can fill itself. if api.eth.SyncMode() != downloader.FullSync { - return api.delayPayloadImport(block) + return api.delayPayloadImport(block), nil } if !api.eth.BlockChain().HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { api.remoteBlocks.put(block.Hash(), block.Header()) @@ -619,11 +619,11 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe // either via a forkchoice update or a sync extension. This method is meant to // be called by the newpayload command when the block seems to be ok, but some // prerequisite prevents it from being processed (e.g. no parent, or snap sync). -func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) delayPayloadImport(block *types.Block) engine.PayloadStatusV1 { // Sanity check that this block's parent is not on a previously invalidated // chain. If it is, mark the block as invalid too. if res := api.checkInvalidAncestor(block.ParentHash(), block.Hash()); res != nil { - return *res, nil + return *res } // Stash the block away for a potential forced forkchoice update to it // at a later time. @@ -635,7 +635,7 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (engine.PayloadS err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()) if err == nil { log.Debug("Payload accepted for sync extension", "number", block.NumberU64(), "hash", block.Hash()) - return engine.PayloadStatusV1{Status: engine.SYNCING}, nil + return engine.PayloadStatusV1{Status: engine.SYNCING} } // Either no beacon sync was started yet, or it rejected the delivered // payload as non-integratable on top of the existing sync. We'll just @@ -652,7 +652,7 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (engine.PayloadS // and cannot afford concurrent out-if-band modifications via imports. log.Warn("Ignoring payload while snap syncing", "number", block.NumberU64(), "hash", block.Hash(), "reason", err) } - return engine.PayloadStatusV1{Status: engine.SYNCING}, nil + return engine.PayloadStatusV1{Status: engine.SYNCING} } // setInvalidAncestor is a callback for the downloader to notify us if a bad block