diff --git a/.mailmap b/.mailmap index aa074b76d6b5..312e51d8544f 100644 --- a/.mailmap +++ b/.mailmap @@ -56,7 +56,6 @@ Diederik Loerakker Dimitry Khokhlov Domino Valdano -Domino Valdano Edgar Aroutiounian diff --git a/accounts/abi/type.go b/accounts/abi/type.go index 2eee11787fda..383982663331 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -179,9 +179,6 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty return Type{}, errors.New("abi: purely anonymous or underscored field is not supported") } fieldName := ResolveNameConflict(name, func(s string) bool { return used[s] }) - if err != nil { - return Type{}, err - } used[fieldName] = true if !isValidFieldName(fieldName) { return Type{}, fmt.Errorf("field %d has invalid name", idx) diff --git a/accounts/accounts.go b/accounts/accounts.go index 6c351a9649ea..b995498a6db9 100644 --- a/accounts/accounts.go +++ b/accounts/accounts.go @@ -195,7 +195,7 @@ func TextHash(data []byte) []byte { // // This gives context to the signed message and prevents signing of transactions. func TextAndHash(data []byte) ([]byte, string) { - msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), string(data)) + msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data) hasher := sha3.NewLegacyKeccak256() hasher.Write([]byte(msg)) return hasher.Sum(nil), msg diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index 48a238048fec..bb92cc2adca5 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -86,7 +86,7 @@ func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error { func TestWatchNewFile(t *testing.T) { t.Parallel() - dir, ks := tmpKeyStore(t, false) + dir, ks := tmpKeyStore(t) // Ensure the watcher is started before adding any files. ks.Accounts() diff --git a/accounts/keystore/keystore.go b/accounts/keystore/keystore.go index 0ffcf376a5fd..e62a8eb25738 100644 --- a/accounts/keystore/keystore.go +++ b/accounts/keystore/keystore.go @@ -87,15 +87,6 @@ func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore { return ks } -// NewPlaintextKeyStore creates a keystore for the given directory. -// Deprecated: Use NewKeyStore. -func NewPlaintextKeyStore(keydir string) *KeyStore { - keydir, _ = filepath.Abs(keydir) - ks := &KeyStore{storage: &keyStorePlain{keydir}} - ks.init(keydir) - return ks -} - func (ks *KeyStore) init(keydir string) { // Lock the mutex since the account cache might call back with events ks.mu.Lock() diff --git a/accounts/keystore/keystore_test.go b/accounts/keystore/keystore_test.go index c9a23eddd6ca..c871392b82ec 100644 --- a/accounts/keystore/keystore_test.go +++ b/accounts/keystore/keystore_test.go @@ -37,7 +37,7 @@ var testSigData = make([]byte, 32) func TestKeyStore(t *testing.T) { t.Parallel() - dir, ks := tmpKeyStore(t, true) + dir, ks := tmpKeyStore(t) a, err := ks.NewAccount("foo") if err != nil { @@ -72,7 +72,7 @@ func TestKeyStore(t *testing.T) { func TestSign(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) pass := "" // not used but required by API a1, err := ks.NewAccount(pass) @@ -89,7 +89,7 @@ func TestSign(t *testing.T) { func TestSignWithPassphrase(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) pass := "passwd" acc, err := ks.NewAccount(pass) @@ -117,7 +117,7 @@ func TestSignWithPassphrase(t *testing.T) { func TestTimedUnlock(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) pass := "foo" a1, err := ks.NewAccount(pass) @@ -152,7 +152,7 @@ func TestTimedUnlock(t *testing.T) { func TestOverrideUnlock(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) pass := "foo" a1, err := ks.NewAccount(pass) @@ -193,7 +193,7 @@ func TestOverrideUnlock(t *testing.T) { // This test should fail under -race if signing races the expiration goroutine. func TestSignRace(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) // Create a test account. a1, err := ks.NewAccount("") @@ -238,7 +238,7 @@ func waitForKsUpdating(t *testing.T, ks *KeyStore, wantStatus bool, maxTime time func TestWalletNotifierLifecycle(t *testing.T) { t.Parallel() // Create a temporary keystore to test with - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) // Ensure that the notification updater is not running yet time.Sleep(250 * time.Millisecond) @@ -284,7 +284,7 @@ type walletEvent struct { // or deleted from the keystore. func TestWalletNotifications(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, false) + _, ks := tmpKeyStore(t) // Subscribe to the wallet feed and collect events. var ( @@ -346,7 +346,7 @@ func TestWalletNotifications(t *testing.T) { // TestImportExport tests the import functionality of a keystore. func TestImportECDSA(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) key, err := crypto.GenerateKey() if err != nil { t.Fatalf("failed to generate key: %v", key) @@ -365,7 +365,7 @@ func TestImportECDSA(t *testing.T) { // TestImportECDSA tests the import and export functionality of a keystore. func TestImportExport(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) acc, err := ks.NewAccount("old") if err != nil { t.Fatalf("failed to create account: %v", acc) @@ -374,7 +374,7 @@ func TestImportExport(t *testing.T) { if err != nil { t.Fatalf("failed to export account: %v", acc) } - _, ks2 := tmpKeyStore(t, true) + _, ks2 := tmpKeyStore(t) if _, err = ks2.Import(json, "old", "old"); err == nil { t.Errorf("importing with invalid password succeeded") } @@ -394,7 +394,7 @@ func TestImportExport(t *testing.T) { // This test should fail under -race if importing races. func TestImportRace(t *testing.T) { t.Parallel() - _, ks := tmpKeyStore(t, true) + _, ks := tmpKeyStore(t) acc, err := ks.NewAccount("old") if err != nil { t.Fatalf("failed to create account: %v", acc) @@ -403,7 +403,7 @@ func TestImportRace(t *testing.T) { if err != nil { t.Fatalf("failed to export account: %v", acc) } - _, ks2 := tmpKeyStore(t, true) + _, ks2 := tmpKeyStore(t) var atom atomic.Uint32 var wg sync.WaitGroup wg.Add(2) @@ -457,11 +457,7 @@ func checkEvents(t *testing.T, want []walletEvent, have []walletEvent) { } } -func tmpKeyStore(t *testing.T, encrypted bool) (string, *KeyStore) { +func tmpKeyStore(t *testing.T) (string, *KeyStore) { d := t.TempDir() - newKs := NewPlaintextKeyStore - if encrypted { - newKs = func(kd string) *KeyStore { return NewKeyStore(kd, veryLightScryptN, veryLightScryptP) } - } - return d, newKs(d) + return d, NewKeyStore(d, veryLightScryptN, veryLightScryptP) } diff --git a/accounts/usbwallet/hub.go b/accounts/usbwallet/hub.go index e67942dbc107..e22dffe9718e 100644 --- a/accounts/usbwallet/hub.go +++ b/accounts/usbwallet/hub.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" - "github.com/karalabe/usb" + "github.com/karalabe/hid" ) // LedgerScheme is the protocol scheme prefixing account and wallet URLs. @@ -109,7 +109,7 @@ func NewTrezorHubWithWebUSB() (*Hub, error) { // newHub creates a new hardware wallet manager for generic USB devices. func newHub(scheme string, vendorID uint16, productIDs []uint16, usageID uint16, endpointID int, makeDriver func(log.Logger) driver) (*Hub, error) { - if !usb.Supported() { + if !hid.Supported() { return nil, errors.New("unsupported platform") } hub := &Hub{ @@ -155,7 +155,7 @@ func (hub *Hub) refreshWallets() { return } // Retrieve the current list of USB wallet devices - var devices []usb.DeviceInfo + var devices []hid.DeviceInfo if runtime.GOOS == "linux" { // hidapi on Linux opens the device during enumeration to retrieve some infos, @@ -170,7 +170,7 @@ func (hub *Hub) refreshWallets() { return } } - infos, err := usb.Enumerate(hub.vendorID, 0) + infos, err := hid.Enumerate(hub.vendorID, 0) if err != nil { failcount := hub.enumFails.Add(1) if runtime.GOOS == "linux" { diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index d0cb93e74e00..81836b3717ac 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -16,7 +16,7 @@ // This file contains the implementation for interacting with the Ledger hardware // wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo: -// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc +// https://github.com/LedgerHQ/app-ethereum/blob/develop/doc/ethapp.adoc package usbwallet diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 69083dc8939d..0fd0415a9ef8 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/karalabe/usb" + "github.com/karalabe/hid" ) // Maximum time between wallet health checks to detect USB unplugs. @@ -79,8 +79,8 @@ type wallet struct { driver driver // Hardware implementation of the low level device operations url *accounts.URL // Textual URL uniquely identifying this wallet - info usb.DeviceInfo // Known USB device infos about the wallet - device usb.Device // USB device advertising itself as a hardware wallet + info hid.DeviceInfo // Known USB device infos about the wallet + device hid.Device // USB device advertising itself as a hardware wallet accounts []accounts.Account // List of derive accounts pinned on the hardware wallet paths map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations diff --git a/appveyor.yml b/appveyor.yml index 4a8c4b737a2f..41c70491b4eb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,7 +26,7 @@ for: - go run build/ci.go lint - go run build/ci.go install -dlgo test_script: - - go run build/ci.go test -dlgo + - go run build/ci.go test -dlgo -short # linux/386 is disabled. - matrix: diff --git a/beacon/blsync/block_sync.go b/beacon/blsync/block_sync.go new file mode 100755 index 000000000000..91b21163e655 --- /dev/null +++ b/beacon/blsync/block_sync.go @@ -0,0 +1,203 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + ctypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/zrnt/eth2/configs" + "github.com/protolambda/ztyp/tree" +) + +// beaconBlockSync implements request.Module; it fetches the beacon blocks belonging +// to the validated and prefetch heads. +type beaconBlockSync struct { + recentBlocks *lru.Cache[common.Hash, *capella.BeaconBlock] + locked map[common.Hash]request.ServerAndID + serverHeads map[request.Server]common.Hash + headTracker headTracker + + lastHeadInfo types.HeadInfo + chainHeadFeed *event.Feed +} + +type headTracker interface { + PrefetchHead() types.HeadInfo + ValidatedHead() (types.SignedHeader, bool) + ValidatedFinality() (types.FinalityUpdate, bool) +} + +// newBeaconBlockSync returns a new beaconBlockSync. +func newBeaconBlockSync(headTracker headTracker, chainHeadFeed *event.Feed) *beaconBlockSync { + return &beaconBlockSync{ + headTracker: headTracker, + chainHeadFeed: chainHeadFeed, + recentBlocks: lru.NewCache[common.Hash, *capella.BeaconBlock](10), + locked: make(map[common.Hash]request.ServerAndID), + serverHeads: make(map[request.Server]common.Hash), + } +} + +// Process implements request.Module. +func (s *beaconBlockSync) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + switch event.Type { + case request.EvResponse, request.EvFail, request.EvTimeout: + sid, req, resp := event.RequestInfo() + blockRoot := common.Hash(req.(sync.ReqBeaconBlock)) + if resp != nil { + s.recentBlocks.Add(blockRoot, resp.(*capella.BeaconBlock)) + } + if s.locked[blockRoot] == sid { + delete(s.locked, blockRoot) + } + case sync.EvNewHead: + s.serverHeads[event.Server] = event.Data.(types.HeadInfo).BlockRoot + case request.EvUnregistered: + delete(s.serverHeads, event.Server) + } + } + s.updateEventFeed() + // request validated head block if unavailable and not yet requested + if vh, ok := s.headTracker.ValidatedHead(); ok { + s.tryRequestBlock(requester, vh.Header.Hash(), false) + } + // request prefetch head if the given server has announced it + if prefetchHead := s.headTracker.PrefetchHead().BlockRoot; prefetchHead != (common.Hash{}) { + s.tryRequestBlock(requester, prefetchHead, true) + } +} + +func (s *beaconBlockSync) tryRequestBlock(requester request.Requester, blockRoot common.Hash, needSameHead bool) { + if _, ok := s.recentBlocks.Get(blockRoot); ok { + return + } + if _, ok := s.locked[blockRoot]; ok { + return + } + for _, server := range requester.CanSendTo() { + if needSameHead && (s.serverHeads[server] != blockRoot) { + continue + } + id := requester.Send(server, sync.ReqBeaconBlock(blockRoot)) + s.locked[blockRoot] = request.ServerAndID{Server: server, ID: id} + return + } +} + +func blockHeadInfo(block *capella.BeaconBlock) types.HeadInfo { + if block == nil { + return types.HeadInfo{} + } + return types.HeadInfo{Slot: uint64(block.Slot), BlockRoot: beaconBlockHash(block)} +} + +// beaconBlockHash calculates the hash of a beacon block. +func beaconBlockHash(beaconBlock *capella.BeaconBlock) common.Hash { + return common.Hash(beaconBlock.HashTreeRoot(configs.Mainnet, tree.GetHashFn())) +} + +// getExecBlock extracts the execution block from the beacon block's payload. +func getExecBlock(beaconBlock *capella.BeaconBlock) (*ctypes.Block, error) { + payload := &beaconBlock.Body.ExecutionPayload + txs := make([]*ctypes.Transaction, len(payload.Transactions)) + for i, opaqueTx := range payload.Transactions { + var tx ctypes.Transaction + if err := tx.UnmarshalBinary(opaqueTx); err != nil { + return nil, fmt.Errorf("failed to parse tx %d: %v", i, err) + } + txs[i] = &tx + } + withdrawals := make([]*ctypes.Withdrawal, len(payload.Withdrawals)) + for i, w := range payload.Withdrawals { + withdrawals[i] = &ctypes.Withdrawal{ + Index: uint64(w.Index), + Validator: uint64(w.ValidatorIndex), + Address: common.Address(w.Address), + Amount: uint64(w.Amount), + } + } + wroot := ctypes.DeriveSha(ctypes.Withdrawals(withdrawals), trie.NewStackTrie(nil)) + execHeader := &ctypes.Header{ + ParentHash: common.Hash(payload.ParentHash), + UncleHash: ctypes.EmptyUncleHash, + Coinbase: common.Address(payload.FeeRecipient), + Root: common.Hash(payload.StateRoot), + TxHash: ctypes.DeriveSha(ctypes.Transactions(txs), trie.NewStackTrie(nil)), + ReceiptHash: common.Hash(payload.ReceiptsRoot), + Bloom: ctypes.Bloom(payload.LogsBloom), + Difficulty: common.Big0, + Number: new(big.Int).SetUint64(uint64(payload.BlockNumber)), + GasLimit: uint64(payload.GasLimit), + GasUsed: uint64(payload.GasUsed), + Time: uint64(payload.Timestamp), + Extra: []byte(payload.ExtraData), + MixDigest: common.Hash(payload.PrevRandao), // reused in merge + Nonce: ctypes.BlockNonce{}, // zero + BaseFee: (*uint256.Int)(&payload.BaseFeePerGas).ToBig(), + WithdrawalsHash: &wroot, + } + execBlock := ctypes.NewBlockWithHeader(execHeader).WithBody(txs, nil).WithWithdrawals(withdrawals) + if execBlockHash := execBlock.Hash(); execBlockHash != common.Hash(payload.BlockHash) { + return execBlock, fmt.Errorf("Sanity check failed, payload hash does not match (expected %x, got %x)", common.Hash(payload.BlockHash), execBlockHash) + } + return execBlock, nil +} + +func (s *beaconBlockSync) updateEventFeed() { + head, ok := s.headTracker.ValidatedHead() + if !ok { + return + } + finality, ok := s.headTracker.ValidatedFinality() //TODO fetch directly if subscription does not deliver + if !ok || head.Header.Epoch() != finality.Attested.Header.Epoch() { + return + } + validatedHead := head.Header.Hash() + headBlock, ok := s.recentBlocks.Get(validatedHead) + if !ok { + return + } + headInfo := blockHeadInfo(headBlock) + if headInfo == s.lastHeadInfo { + return + } + s.lastHeadInfo = headInfo + // new head block and finality info available; extract executable data and send event to feed + execBlock, err := getExecBlock(headBlock) + if err != nil { + log.Error("Error extracting execution block from validated beacon block", "error", err) + return + } + s.chainHeadFeed.Send(types.ChainHeadEvent{ + HeadBlock: engine.BlockToExecutableData(execBlock, nil, nil).ExecutionPayload, + Finalized: common.Hash(finality.Finalized.PayloadHeader.BlockHash), + }) +} diff --git a/beacon/blsync/block_sync_test.go b/beacon/blsync/block_sync_test.go new file mode 100644 index 000000000000..9ce434d86273 --- /dev/null +++ b/beacon/blsync/block_sync_test.go @@ -0,0 +1,160 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "testing" + + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/event" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/zrnt/eth2/configs" + "github.com/protolambda/ztyp/tree" +) + +var ( + testServer1 = "testServer1" + testServer2 = "testServer2" + + testBlock1 = &capella.BeaconBlock{ + Slot: 123, + Body: capella.BeaconBlockBody{ + ExecutionPayload: capella.ExecutionPayload{BlockNumber: 456}, + }, + } + testBlock2 = &capella.BeaconBlock{ + Slot: 124, + Body: capella.BeaconBlockBody{ + ExecutionPayload: capella.ExecutionPayload{BlockNumber: 457}, + }, + } +) + +func init() { + eb1, _ := getExecBlock(testBlock1) + testBlock1.Body.ExecutionPayload.BlockHash = tree.Root(eb1.Hash()) + eb2, _ := getExecBlock(testBlock2) + testBlock2.Body.ExecutionPayload.BlockHash = tree.Root(eb2.Hash()) +} + +func TestBlockSync(t *testing.T) { + ht := &testHeadTracker{} + eventFeed := new(event.Feed) + blockSync := newBeaconBlockSync(ht, eventFeed) + headCh := make(chan types.ChainHeadEvent, 16) + eventFeed.Subscribe(headCh) + ts := sync.NewTestScheduler(t, blockSync) + ts.AddServer(testServer1, 1) + ts.AddServer(testServer2, 1) + + expHeadBlock := func(tci int, expHead *capella.BeaconBlock) { + var expNumber, headNumber uint64 + if expHead != nil { + expNumber = uint64(expHead.Body.ExecutionPayload.BlockNumber) + } + select { + case event := <-headCh: + headNumber = event.HeadBlock.Number + default: + } + if headNumber != expNumber { + t.Errorf("Wrong head block in test case #%d (expected block number %d, got %d)", tci, expNumber, headNumber) + } + } + + // no block requests expected until head tracker knows about a head + ts.Run(1) + expHeadBlock(1, nil) + + // set block 1 as prefetch head, announced by server 2 + head1 := blockHeadInfo(testBlock1) + ht.prefetch = head1 + ts.ServerEvent(sync.EvNewHead, testServer2, head1) + // expect request to server 2 which has announced the head + ts.Run(2, testServer2, sync.ReqBeaconBlock(head1.BlockRoot)) + + // valid response + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testBlock1) + ts.AddAllowance(testServer2, 1) + ts.Run(3) + // head block still not expected as the fetched block is not the validated head yet + expHeadBlock(3, nil) + + // set as validated head, expect no further requests but block 1 set as head block + ht.validated.Header = blockHeader(testBlock1) + ts.Run(4) + expHeadBlock(4, testBlock1) + + // set block 2 as prefetch head, announced by server 1 + head2 := blockHeadInfo(testBlock2) + ht.prefetch = head2 + ts.ServerEvent(sync.EvNewHead, testServer1, head2) + // expect request to server 1 + ts.Run(5, testServer1, sync.ReqBeaconBlock(head2.BlockRoot)) + + // req2 fails, no further requests expected because server 2 has not announced it + ts.RequestEvent(request.EvFail, ts.Request(5, 1), nil) + ts.Run(6) + + // set as validated head before retrieving block; now it's assumed to be available from server 2 too + ht.validated.Header = blockHeader(testBlock2) + // expect req2 retry to server 2 + ts.Run(7, testServer2, sync.ReqBeaconBlock(head2.BlockRoot)) + // now head block should be unavailable again + expHeadBlock(4, nil) + + // valid response, now head block should be block 2 immediately as it is already validated + ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testBlock2) + ts.Run(8) + expHeadBlock(5, testBlock2) +} + +func blockHeader(block *capella.BeaconBlock) types.Header { + return types.Header{ + Slot: uint64(block.Slot), + ProposerIndex: uint64(block.ProposerIndex), + ParentRoot: common.Hash(block.ParentRoot), + StateRoot: common.Hash(block.StateRoot), + BodyRoot: common.Hash(block.Body.HashTreeRoot(configs.Mainnet, tree.GetHashFn())), + } +} + +type testHeadTracker struct { + prefetch types.HeadInfo + validated types.SignedHeader +} + +func (h *testHeadTracker) PrefetchHead() types.HeadInfo { + return h.prefetch +} + +func (h *testHeadTracker) ValidatedHead() (types.SignedHeader, bool) { + return h.validated, h.validated.Header != (types.Header{}) +} + +// TODO add test case for finality +func (h *testHeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) { + return types.FinalityUpdate{ + Attested: types.HeaderWithExecProof{Header: h.validated.Header}, + Finalized: types.HeaderWithExecProof{PayloadHeader: &capella.ExecutionPayloadHeader{}}, + Signature: h.validated.Signature, + SignatureSlot: h.validated.SignatureSlot, + }, h.validated.Header != (types.Header{}) +} diff --git a/beacon/blsync/client.go b/beacon/blsync/client.go new file mode 100644 index 000000000000..1bfbb1316069 --- /dev/null +++ b/beacon/blsync/client.go @@ -0,0 +1,103 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "strings" + + "github.com/ethereum/go-ethereum/beacon/light" + "github.com/ethereum/go-ethereum/beacon/light/api" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/event" + "github.com/urfave/cli/v2" +) + +type Client struct { + scheduler *request.Scheduler + chainHeadFeed *event.Feed + urls []string + customHeader map[string]string +} + +func NewClient(ctx *cli.Context) *Client { + if !ctx.IsSet(utils.BeaconApiFlag.Name) { + utils.Fatalf("Beacon node light client API URL not specified") + } + var ( + chainConfig = makeChainConfig(ctx) + customHeader = make(map[string]string) + ) + for _, s := range ctx.StringSlice(utils.BeaconApiHeaderFlag.Name) { + kv := strings.Split(s, ":") + if len(kv) != 2 { + utils.Fatalf("Invalid custom API header entry: %s", s) + } + customHeader[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) + } + // create data structures + var ( + db = memorydb.New() + threshold = ctx.Int(utils.BeaconThresholdFlag.Name) + committeeChain = light.NewCommitteeChain(db, chainConfig.ChainConfig, threshold, !ctx.Bool(utils.BeaconNoFilterFlag.Name)) + headTracker = light.NewHeadTracker(committeeChain, threshold) + ) + headSync := sync.NewHeadSync(headTracker, committeeChain) + + // set up scheduler and sync modules + chainHeadFeed := new(event.Feed) + scheduler := request.NewScheduler() + checkpointInit := sync.NewCheckpointInit(committeeChain, chainConfig.Checkpoint) + forwardSync := sync.NewForwardUpdateSync(committeeChain) + beaconBlockSync := newBeaconBlockSync(headTracker, chainHeadFeed) + scheduler.RegisterTarget(headTracker) + scheduler.RegisterTarget(committeeChain) + scheduler.RegisterModule(checkpointInit, "checkpointInit") + scheduler.RegisterModule(forwardSync, "forwardSync") + scheduler.RegisterModule(headSync, "headSync") + scheduler.RegisterModule(beaconBlockSync, "beaconBlockSync") + + return &Client{ + scheduler: scheduler, + urls: ctx.StringSlice(utils.BeaconApiFlag.Name), + customHeader: customHeader, + chainHeadFeed: chainHeadFeed, + } +} + +// SubscribeChainHeadEvent allows callers to subscribe a provided channel to new +// head updates. +func (c *Client) SubscribeChainHeadEvent(ch chan<- types.ChainHeadEvent) event.Subscription { + return c.chainHeadFeed.Subscribe(ch) +} + +func (c *Client) Start() { + c.scheduler.Start() + // register server(s) + for _, url := range c.urls { + beaconApi := api.NewBeaconLightApi(url, c.customHeader) + c.scheduler.RegisterServer(request.NewServer(api.NewApiServer(beaconApi), &mclock.System{})) + } +} + +func (c *Client) Stop() { + c.scheduler.Stop() +} diff --git a/beacon/blsync/config.go b/beacon/blsync/config.go new file mode 100644 index 000000000000..b51d3e2b0566 --- /dev/null +++ b/beacon/blsync/config.go @@ -0,0 +1,113 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package blsync + +import ( + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/urfave/cli/v2" +) + +// lightClientConfig contains beacon light client configuration +type lightClientConfig struct { + *types.ChainConfig + Checkpoint common.Hash +} + +var ( + MainnetConfig = lightClientConfig{ + ChainConfig: (&types.ChainConfig{ + GenesisValidatorsRoot: common.HexToHash("0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"), + GenesisTime: 1606824023, + }). + AddFork("GENESIS", 0, []byte{0, 0, 0, 0}). + AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}). + AddFork("BELLATRIX", 144896, []byte{2, 0, 0, 0}). + AddFork("CAPELLA", 194048, []byte{3, 0, 0, 0}), + Checkpoint: common.HexToHash("0x388be41594ec7d6a6894f18c73f3469f07e2c19a803de4755d335817ed8e2e5a"), + } + + SepoliaConfig = lightClientConfig{ + ChainConfig: (&types.ChainConfig{ + GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"), + GenesisTime: 1655733600, + }). + AddFork("GENESIS", 0, []byte{144, 0, 0, 105}). + AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}). + AddFork("BELLATRIX", 100, []byte{144, 0, 0, 113}). + AddFork("CAPELLA", 56832, []byte{144, 0, 0, 114}), + Checkpoint: common.HexToHash("0x1005a6d9175e96bfbce4d35b80f468e9bff0b674e1e861d16e09e10005a58e81"), + } + + GoerliConfig = lightClientConfig{ + ChainConfig: (&types.ChainConfig{ + GenesisValidatorsRoot: common.HexToHash("0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb"), + GenesisTime: 1614588812, + }). + AddFork("GENESIS", 0, []byte{0, 0, 16, 32}). + AddFork("ALTAIR", 36660, []byte{1, 0, 16, 32}). + AddFork("BELLATRIX", 112260, []byte{2, 0, 16, 32}). + AddFork("CAPELLA", 162304, []byte{3, 0, 16, 32}), + Checkpoint: common.HexToHash("0x53a0f4f0a378e2c4ae0a9ee97407eb69d0d737d8d8cd0a5fb1093f42f7b81c49"), + } +) + +func makeChainConfig(ctx *cli.Context) lightClientConfig { + utils.CheckExclusive(ctx, utils.MainnetFlag, utils.GoerliFlag, utils.SepoliaFlag) + customConfig := ctx.IsSet(utils.BeaconConfigFlag.Name) || ctx.IsSet(utils.BeaconGenesisRootFlag.Name) || ctx.IsSet(utils.BeaconGenesisTimeFlag.Name) + var config lightClientConfig + switch { + case ctx.Bool(utils.MainnetFlag.Name): + config = MainnetConfig + case ctx.Bool(utils.SepoliaFlag.Name): + config = SepoliaConfig + case ctx.Bool(utils.GoerliFlag.Name): + config = GoerliConfig + default: + if !customConfig { + config = MainnetConfig + } + } + if customConfig && config.Forks != nil { + utils.Fatalf("Cannot use custom beacon chain config flags in combination with pre-defined network config") + } + if ctx.IsSet(utils.BeaconGenesisRootFlag.Name) { + if c, err := hexutil.Decode(ctx.String(utils.BeaconGenesisRootFlag.Name)); err == nil && len(c) <= 32 { + copy(config.GenesisValidatorsRoot[:len(c)], c) + } else { + utils.Fatalf("Invalid hex string", "beacon.genesis.gvroot", ctx.String(utils.BeaconGenesisRootFlag.Name), "error", err) + } + } + if ctx.IsSet(utils.BeaconGenesisTimeFlag.Name) { + config.GenesisTime = ctx.Uint64(utils.BeaconGenesisTimeFlag.Name) + } + if ctx.IsSet(utils.BeaconConfigFlag.Name) { + if err := config.ChainConfig.LoadForks(ctx.String(utils.BeaconConfigFlag.Name)); err != nil { + utils.Fatalf("Could not load beacon chain config file", "file name", ctx.String(utils.BeaconConfigFlag.Name), "error", err) + } + } + if ctx.IsSet(utils.BeaconCheckpointFlag.Name) { + if c, err := hexutil.Decode(ctx.String(utils.BeaconCheckpointFlag.Name)); err == nil && len(c) <= 32 { + copy(config.Checkpoint[:len(c)], c) + } else { + utils.Fatalf("Invalid hex string", "beacon.checkpoint", ctx.String(utils.BeaconCheckpointFlag.Name), "error", err) + } + } + return config +} diff --git a/beacon/light/api/api_server.go b/beacon/light/api/api_server.go new file mode 100755 index 000000000000..da044f4b2d6e --- /dev/null +++ b/beacon/light/api/api_server.go @@ -0,0 +1,103 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "reflect" + + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/light/sync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +// ApiServer is a wrapper around BeaconLightApi that implements request.requestServer. +type ApiServer struct { + api *BeaconLightApi + eventCallback func(event request.Event) + unsubscribe func() +} + +// NewApiServer creates a new ApiServer. +func NewApiServer(api *BeaconLightApi) *ApiServer { + return &ApiServer{api: api} +} + +// Subscribe implements request.requestServer. +func (s *ApiServer) Subscribe(eventCallback func(event request.Event)) { + s.eventCallback = eventCallback + listener := HeadEventListener{ + OnNewHead: func(slot uint64, blockRoot common.Hash) { + log.Debug("New head received", "slot", slot, "blockRoot", blockRoot) + eventCallback(request.Event{Type: sync.EvNewHead, Data: types.HeadInfo{Slot: slot, BlockRoot: blockRoot}}) + }, + OnSignedHead: func(head types.SignedHeader) { + log.Debug("New signed head received", "slot", head.Header.Slot, "blockRoot", head.Header.Hash(), "signerCount", head.Signature.SignerCount()) + eventCallback(request.Event{Type: sync.EvNewSignedHead, Data: head}) + }, + OnFinality: func(head types.FinalityUpdate) { + log.Debug("New finality update received", "slot", head.Attested.Slot, "blockRoot", head.Attested.Hash(), "signerCount", head.Signature.SignerCount()) + eventCallback(request.Event{Type: sync.EvNewFinalityUpdate, Data: head}) + }, + OnError: func(err error) { + log.Warn("Head event stream error", "err", err) + }, + } + s.unsubscribe = s.api.StartHeadListener(listener) +} + +// SendRequest implements request.requestServer. +func (s *ApiServer) SendRequest(id request.ID, req request.Request) { + go func() { + var resp request.Response + var err error + switch data := req.(type) { + case sync.ReqUpdates: + log.Debug("Beacon API: requesting light client update", "reqid", id, "period", data.FirstPeriod, "count", data.Count) + var r sync.RespUpdates + r.Updates, r.Committees, err = s.api.GetBestUpdatesAndCommittees(data.FirstPeriod, data.Count) + resp = r + case sync.ReqHeader: + log.Debug("Beacon API: requesting header", "reqid", id, "hash", common.Hash(data)) + resp, err = s.api.GetHeader(common.Hash(data)) + case sync.ReqCheckpointData: + log.Debug("Beacon API: requesting checkpoint data", "reqid", id, "hash", common.Hash(data)) + resp, err = s.api.GetCheckpointData(common.Hash(data)) + case sync.ReqBeaconBlock: + log.Debug("Beacon API: requesting block", "reqid", id, "hash", common.Hash(data)) + resp, err = s.api.GetBeaconBlock(common.Hash(data)) + default: + } + + if err != nil { + log.Warn("Beacon API request failed", "type", reflect.TypeOf(req), "reqid", id, "err", err) + s.eventCallback(request.Event{Type: request.EvFail, Data: request.RequestResponse{ID: id, Request: req}}) + } else { + s.eventCallback(request.Event{Type: request.EvResponse, Data: request.RequestResponse{ID: id, Request: req, Response: resp}}) + } + }() +} + +// Unsubscribe implements request.requestServer. +// Note: Unsubscribe should not be called concurrently with Subscribe. +func (s *ApiServer) Unsubscribe() { + if s.unsubscribe != nil { + s.unsubscribe() + s.unsubscribe = nil + } +} diff --git a/beacon/light/api/light_api.go b/beacon/light/api/light_api.go new file mode 100755 index 000000000000..fd701dc0a871 --- /dev/null +++ b/beacon/light/api/light_api.go @@ -0,0 +1,496 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more detaiapi. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" + + "github.com/donovanhide/eventsource" + "github.com/ethereum/go-ethereum/beacon/merkle" + "github.com/ethereum/go-ethereum/beacon/params" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/zrnt/eth2/configs" + "github.com/protolambda/ztyp/tree" +) + +var ( + ErrNotFound = errors.New("404 Not Found") + ErrInternal = errors.New("500 Internal Server Error") +) + +type CommitteeUpdate struct { + Version string + Update types.LightClientUpdate + NextSyncCommittee types.SerializedSyncCommittee +} + +// See data structure definition here: +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientupdate +type committeeUpdateJson struct { + Version string `json:"version"` + Data committeeUpdateData `json:"data"` +} + +type committeeUpdateData struct { + Header jsonBeaconHeader `json:"attested_header"` + NextSyncCommittee types.SerializedSyncCommittee `json:"next_sync_committee"` + NextSyncCommitteeBranch merkle.Values `json:"next_sync_committee_branch"` + FinalizedHeader *jsonBeaconHeader `json:"finalized_header,omitempty"` + FinalityBranch merkle.Values `json:"finality_branch,omitempty"` + SyncAggregate types.SyncAggregate `json:"sync_aggregate"` + SignatureSlot common.Decimal `json:"signature_slot"` +} + +type jsonBeaconHeader struct { + Beacon types.Header `json:"beacon"` +} + +type jsonHeaderWithExecProof struct { + Beacon types.Header `json:"beacon"` + Execution *capella.ExecutionPayloadHeader `json:"execution"` + ExecutionBranch merkle.Values `json:"execution_branch"` +} + +// UnmarshalJSON unmarshals from JSON. +func (u *CommitteeUpdate) UnmarshalJSON(input []byte) error { + var dec committeeUpdateJson + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + u.Version = dec.Version + u.NextSyncCommittee = dec.Data.NextSyncCommittee + u.Update = types.LightClientUpdate{ + AttestedHeader: types.SignedHeader{ + Header: dec.Data.Header.Beacon, + Signature: dec.Data.SyncAggregate, + SignatureSlot: uint64(dec.Data.SignatureSlot), + }, + NextSyncCommitteeRoot: u.NextSyncCommittee.Root(), + NextSyncCommitteeBranch: dec.Data.NextSyncCommitteeBranch, + FinalityBranch: dec.Data.FinalityBranch, + } + if dec.Data.FinalizedHeader != nil { + u.Update.FinalizedHeader = &dec.Data.FinalizedHeader.Beacon + } + return nil +} + +// fetcher is an interface useful for debug-harnessing the http api. +type fetcher interface { + Do(req *http.Request) (*http.Response, error) +} + +// BeaconLightApi requests light client information from a beacon node REST API. +// Note: all required API endpoints are currently only implemented by Lodestar. +type BeaconLightApi struct { + url string + client fetcher + customHeaders map[string]string +} + +func NewBeaconLightApi(url string, customHeaders map[string]string) *BeaconLightApi { + return &BeaconLightApi{ + url: url, + client: &http.Client{ + Timeout: time.Second * 10, + }, + customHeaders: customHeaders, + } +} + +func (api *BeaconLightApi) httpGet(path string) ([]byte, error) { + req, err := http.NewRequest("GET", api.url+path, nil) + if err != nil { + return nil, err + } + for k, v := range api.customHeaders { + req.Header.Set(k, v) + } + resp, err := api.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + switch resp.StatusCode { + case 200: + return io.ReadAll(resp.Body) + case 404: + return nil, ErrNotFound + case 500: + return nil, ErrInternal + default: + return nil, fmt.Errorf("unexpected error from API endpoint \"%s\": status code %d", path, resp.StatusCode) + } +} + +func (api *BeaconLightApi) httpGetf(format string, params ...any) ([]byte, error) { + return api.httpGet(fmt.Sprintf(format, params...)) +} + +// GetBestUpdateAndCommittee fetches and validates LightClientUpdate for given +// period and full serialized committee for the next period (committee root hash +// equals update.NextSyncCommitteeRoot). +// Note that the results are validated but the update signature should be verified +// by the caller as its validity depends on the update chain. +func (api *BeaconLightApi) GetBestUpdatesAndCommittees(firstPeriod, count uint64) ([]*types.LightClientUpdate, []*types.SerializedSyncCommittee, error) { + resp, err := api.httpGetf("/eth/v1/beacon/light_client/updates?start_period=%d&count=%d", firstPeriod, count) + if err != nil { + return nil, nil, err + } + + var data []CommitteeUpdate + if err := json.Unmarshal(resp, &data); err != nil { + return nil, nil, err + } + if len(data) != int(count) { + return nil, nil, errors.New("invalid number of committee updates") + } + updates := make([]*types.LightClientUpdate, int(count)) + committees := make([]*types.SerializedSyncCommittee, int(count)) + for i, d := range data { + if d.Update.AttestedHeader.Header.SyncPeriod() != firstPeriod+uint64(i) { + return nil, nil, errors.New("wrong committee update header period") + } + if err := d.Update.Validate(); err != nil { + return nil, nil, err + } + if d.NextSyncCommittee.Root() != d.Update.NextSyncCommitteeRoot { + return nil, nil, errors.New("wrong sync committee root") + } + updates[i], committees[i] = new(types.LightClientUpdate), new(types.SerializedSyncCommittee) + *updates[i], *committees[i] = d.Update, d.NextSyncCommittee + } + return updates, committees, nil +} + +// GetOptimisticHeadUpdate fetches a signed header based on the latest available +// optimistic update. Note that the signature should be verified by the caller +// as its validity depends on the update chain. +// +// See data structure definition here: +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientoptimisticupdate +func (api *BeaconLightApi) GetOptimisticHeadUpdate() (types.SignedHeader, error) { + resp, err := api.httpGet("/eth/v1/beacon/light_client/optimistic_update") + if err != nil { + return types.SignedHeader{}, err + } + return decodeOptimisticHeadUpdate(resp) +} + +func decodeOptimisticHeadUpdate(enc []byte) (types.SignedHeader, error) { + var data struct { + Data struct { + Header jsonBeaconHeader `json:"attested_header"` + Aggregate types.SyncAggregate `json:"sync_aggregate"` + SignatureSlot common.Decimal `json:"signature_slot"` + } `json:"data"` + } + if err := json.Unmarshal(enc, &data); err != nil { + return types.SignedHeader{}, err + } + if data.Data.Header.Beacon.StateRoot == (common.Hash{}) { + // workaround for different event encoding format in Lodestar + if err := json.Unmarshal(enc, &data.Data); err != nil { + return types.SignedHeader{}, err + } + } + + if len(data.Data.Aggregate.Signers) != params.SyncCommitteeBitmaskSize { + return types.SignedHeader{}, errors.New("invalid sync_committee_bits length") + } + if len(data.Data.Aggregate.Signature) != params.BLSSignatureSize { + return types.SignedHeader{}, errors.New("invalid sync_committee_signature length") + } + return types.SignedHeader{ + Header: data.Data.Header.Beacon, + Signature: data.Data.Aggregate, + SignatureSlot: uint64(data.Data.SignatureSlot), + }, nil +} + +// GetFinalityUpdate fetches the latest available finality update. +// +// See data structure definition here: +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientfinalityupdate +func (api *BeaconLightApi) GetFinalityUpdate() (types.FinalityUpdate, error) { + resp, err := api.httpGet("/eth/v1/beacon/light_client/finality_update") + if err != nil { + return types.FinalityUpdate{}, err + } + return decodeFinalityUpdate(resp) +} + +func decodeFinalityUpdate(enc []byte) (types.FinalityUpdate, error) { + var data struct { + Data struct { + Attested jsonHeaderWithExecProof `json:"attested_header"` + Finalized jsonHeaderWithExecProof `json:"finalized_header"` + FinalityBranch merkle.Values `json:"finality_branch"` + Aggregate types.SyncAggregate `json:"sync_aggregate"` + SignatureSlot common.Decimal `json:"signature_slot"` + } `json:"data"` + } + if err := json.Unmarshal(enc, &data); err != nil { + return types.FinalityUpdate{}, err + } + + if len(data.Data.Aggregate.Signers) != params.SyncCommitteeBitmaskSize { + return types.FinalityUpdate{}, errors.New("invalid sync_committee_bits length") + } + if len(data.Data.Aggregate.Signature) != params.BLSSignatureSize { + return types.FinalityUpdate{}, errors.New("invalid sync_committee_signature length") + } + return types.FinalityUpdate{ + Attested: types.HeaderWithExecProof{ + Header: data.Data.Attested.Beacon, + PayloadHeader: data.Data.Attested.Execution, + PayloadBranch: data.Data.Attested.ExecutionBranch, + }, + Finalized: types.HeaderWithExecProof{ + Header: data.Data.Finalized.Beacon, + PayloadHeader: data.Data.Finalized.Execution, + PayloadBranch: data.Data.Finalized.ExecutionBranch, + }, + FinalityBranch: data.Data.FinalityBranch, + Signature: data.Data.Aggregate, + SignatureSlot: uint64(data.Data.SignatureSlot), + }, nil +} + +// GetHead fetches and validates the beacon header with the given blockRoot. +// If blockRoot is null hash then the latest head header is fetched. +func (api *BeaconLightApi) GetHeader(blockRoot common.Hash) (types.Header, error) { + var blockId string + if blockRoot == (common.Hash{}) { + blockId = "head" + } else { + blockId = blockRoot.Hex() + } + resp, err := api.httpGetf("/eth/v1/beacon/headers/%s", blockId) + if err != nil { + return types.Header{}, err + } + + var data struct { + Data struct { + Root common.Hash `json:"root"` + Canonical bool `json:"canonical"` + Header struct { + Message types.Header `json:"message"` + Signature hexutil.Bytes `json:"signature"` + } `json:"header"` + } `json:"data"` + } + if err := json.Unmarshal(resp, &data); err != nil { + return types.Header{}, err + } + header := data.Data.Header.Message + if blockRoot == (common.Hash{}) { + blockRoot = data.Data.Root + } + if header.Hash() != blockRoot { + return types.Header{}, errors.New("retrieved beacon header root does not match") + } + return header, nil +} + +// GetCheckpointData fetches and validates bootstrap data belonging to the given checkpoint. +func (api *BeaconLightApi) GetCheckpointData(checkpointHash common.Hash) (*types.BootstrapData, error) { + resp, err := api.httpGetf("/eth/v1/beacon/light_client/bootstrap/0x%x", checkpointHash[:]) + if err != nil { + return nil, err + } + + // See data structure definition here: + // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientbootstrap + type bootstrapData struct { + Data struct { + Header jsonBeaconHeader `json:"header"` + Committee *types.SerializedSyncCommittee `json:"current_sync_committee"` + CommitteeBranch merkle.Values `json:"current_sync_committee_branch"` + } `json:"data"` + } + + var data bootstrapData + if err := json.Unmarshal(resp, &data); err != nil { + return nil, err + } + if data.Data.Committee == nil { + return nil, errors.New("sync committee is missing") + } + header := data.Data.Header.Beacon + if header.Hash() != checkpointHash { + return nil, fmt.Errorf("invalid checkpoint block header, have %v want %v", header.Hash(), checkpointHash) + } + checkpoint := &types.BootstrapData{ + Header: header, + CommitteeBranch: data.Data.CommitteeBranch, + CommitteeRoot: data.Data.Committee.Root(), + Committee: data.Data.Committee, + } + if err := checkpoint.Validate(); err != nil { + return nil, fmt.Errorf("invalid checkpoint: %w", err) + } + if checkpoint.Header.Hash() != checkpointHash { + return nil, errors.New("wrong checkpoint hash") + } + return checkpoint, nil +} + +func (api *BeaconLightApi) GetBeaconBlock(blockRoot common.Hash) (*capella.BeaconBlock, error) { + resp, err := api.httpGetf("/eth/v2/beacon/blocks/0x%x", blockRoot) + if err != nil { + return nil, err + } + + var beaconBlockMessage struct { + Data struct { + Message capella.BeaconBlock `json:"message"` + } `json:"data"` + } + if err := json.Unmarshal(resp, &beaconBlockMessage); err != nil { + return nil, fmt.Errorf("invalid block json data: %v", err) + } + beaconBlock := new(capella.BeaconBlock) + *beaconBlock = beaconBlockMessage.Data.Message + root := common.Hash(beaconBlock.HashTreeRoot(configs.Mainnet, tree.GetHashFn())) + if root != blockRoot { + return nil, fmt.Errorf("Beacon block root hash mismatch (expected: %x, got: %x)", blockRoot, root) + } + return beaconBlock, nil +} + +func decodeHeadEvent(enc []byte) (uint64, common.Hash, error) { + var data struct { + Slot common.Decimal `json:"slot"` + Block common.Hash `json:"block"` + } + if err := json.Unmarshal(enc, &data); err != nil { + return 0, common.Hash{}, err + } + return uint64(data.Slot), data.Block, nil +} + +type HeadEventListener struct { + OnNewHead func(slot uint64, blockRoot common.Hash) + OnSignedHead func(head types.SignedHeader) + OnFinality func(head types.FinalityUpdate) + OnError func(err error) +} + +// StartHeadListener creates an event subscription for heads and signed (optimistic) +// head updates and calls the specified callback functions when they are received. +// The callbacks are also called for the current head and optimistic head at startup. +// They are never called concurrently. +func (api *BeaconLightApi) StartHeadListener(listener HeadEventListener) func() { + closeCh := make(chan struct{}) // initiate closing the stream + closedCh := make(chan struct{}) // stream closed (or failed to create) + stoppedCh := make(chan struct{}) // sync loop stopped + streamCh := make(chan *eventsource.Stream, 1) + go func() { + defer close(closedCh) + // when connected to a Lodestar node the subscription blocks until the + // first actual event arrives; therefore we create the subscription in + // a separate goroutine while letting the main goroutine sync up to the + // current head + req, err := http.NewRequest("GET", api.url+ + "/eth/v1/events?topics=head&topics=light_client_optimistic_update&topics=light_client_finality_update", nil) + if err != nil { + listener.OnError(fmt.Errorf("error creating event subscription request: %v", err)) + return + } + for k, v := range api.customHeaders { + req.Header.Set(k, v) + } + stream, err := eventsource.SubscribeWithRequest("", req) + if err != nil { + listener.OnError(fmt.Errorf("error creating event subscription: %v", err)) + close(streamCh) + return + } + streamCh <- stream + <-closeCh + stream.Close() + }() + + go func() { + defer close(stoppedCh) + + if head, err := api.GetHeader(common.Hash{}); err == nil { + listener.OnNewHead(head.Slot, head.Hash()) + } + if signedHead, err := api.GetOptimisticHeadUpdate(); err == nil { + listener.OnSignedHead(signedHead) + } + if finalityUpdate, err := api.GetFinalityUpdate(); err == nil { + listener.OnFinality(finalityUpdate) + } + stream := <-streamCh + if stream == nil { + return + } + + for { + select { + case event, ok := <-stream.Events: + if !ok { + break + } + switch event.Event() { + case "head": + if slot, blockRoot, err := decodeHeadEvent([]byte(event.Data())); err == nil { + listener.OnNewHead(slot, blockRoot) + } else { + listener.OnError(fmt.Errorf("error decoding head event: %v", err)) + } + case "light_client_optimistic_update": + if signedHead, err := decodeOptimisticHeadUpdate([]byte(event.Data())); err == nil { + listener.OnSignedHead(signedHead) + } else { + listener.OnError(fmt.Errorf("error decoding optimistic update event: %v", err)) + } + case "light_client_finality_update": + if finalityUpdate, err := decodeFinalityUpdate([]byte(event.Data())); err == nil { + listener.OnFinality(finalityUpdate) + } else { + listener.OnError(fmt.Errorf("error decoding finality update event: %v", err)) + } + default: + listener.OnError(fmt.Errorf("unexpected event: %s", event.Event())) + } + case err, ok := <-stream.Errors: + if !ok { + break + } + listener.OnError(err) + } + } + }() + return func() { + close(closeCh) + <-closedCh + <-stoppedCh + } +} diff --git a/beacon/light/committee_chain.go b/beacon/light/committee_chain.go index d707f8cc34da..a8d032bb65c5 100644 --- a/beacon/light/committee_chain.go +++ b/beacon/light/committee_chain.go @@ -70,6 +70,7 @@ type CommitteeChain struct { committees *canonicalStore[*types.SerializedSyncCommittee] fixedCommitteeRoots *canonicalStore[common.Hash] committeeCache *lru.Cache[uint64, syncCommittee] // cache deserialized committees + changeCounter uint64 clock mclock.Clock // monotonic clock (simulated clock in tests) unixNano func() int64 // system clock (simulated clock in tests) @@ -86,6 +87,11 @@ func NewCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signer return newCommitteeChain(db, config, signerThreshold, enforceTime, blsVerifier{}, &mclock.System{}, func() int64 { return time.Now().UnixNano() }) } +// NewTestCommitteeChain creates a new CommitteeChain for testing. +func NewTestCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, clock *mclock.Simulated) *CommitteeChain { + return newCommitteeChain(db, config, signerThreshold, enforceTime, dummyVerifier{}, clock, func() int64 { return int64(clock.Now()) }) +} + // newCommitteeChain creates a new CommitteeChain with the option of replacing the // clock source and signature verification for testing purposes. func newCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, sigVerifier committeeSigVerifier, clock mclock.Clock, unixNano func() int64) *CommitteeChain { @@ -181,20 +187,20 @@ func (s *CommitteeChain) Reset() { if err := s.rollback(0); err != nil { log.Error("Error writing batch into chain database", "error", err) } + s.changeCounter++ } -// CheckpointInit initializes a CommitteeChain based on the checkpoint. +// CheckpointInit initializes a CommitteeChain based on a checkpoint. // Note: if the chain is already initialized and the committees proven by the // checkpoint do match the existing chain then the chain is retained and the // new checkpoint becomes fixed. -func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error { +func (s *CommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error { s.chainmu.Lock() defer s.chainmu.Unlock() if err := bootstrap.Validate(); err != nil { return err } - period := bootstrap.Header.SyncPeriod() if err := s.deleteFixedCommitteeRootsFrom(period + 2); err != nil { s.Reset() @@ -215,6 +221,7 @@ func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error { s.Reset() return err } + s.changeCounter++ return nil } @@ -367,6 +374,7 @@ func (s *CommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommi return ErrWrongCommitteeRoot } } + s.changeCounter++ if reorg { if err := s.rollback(period + 1); err != nil { return err @@ -405,6 +413,13 @@ func (s *CommitteeChain) NextSyncPeriod() (uint64, bool) { return s.committees.periods.End - 1, true } +func (s *CommitteeChain) ChangeCounter() uint64 { + s.chainmu.RLock() + defer s.chainmu.RUnlock() + + return s.changeCounter +} + // rollback removes all committees and fixed roots from the given period and updates // starting from the previous period. func (s *CommitteeChain) rollback(period uint64) error { @@ -452,12 +467,12 @@ func (s *CommitteeChain) getSyncCommittee(period uint64) (syncCommittee, error) if sc, ok := s.committees.get(s.db, period); ok { c, err := s.sigVerifier.deserializeSyncCommittee(sc) if err != nil { - return nil, fmt.Errorf("Sync committee #%d deserialization error: %v", period, err) + return nil, fmt.Errorf("sync committee #%d deserialization error: %v", period, err) } s.committeeCache.Add(period, c) return c, nil } - return nil, fmt.Errorf("Missing serialized sync committee #%d", period) + return nil, fmt.Errorf("missing serialized sync committee #%d", period) } // VerifySignedHeader returns true if the given signed header has a valid signature diff --git a/beacon/light/committee_chain_test.go b/beacon/light/committee_chain_test.go index 60ea2a0efdbf..57b6d7175cce 100644 --- a/beacon/light/committee_chain_test.go +++ b/beacon/light/committee_chain_test.go @@ -241,12 +241,12 @@ func newCommitteeChainTest(t *testing.T, config types.ChainConfig, signerThresho signerThreshold: signerThreshold, enforceTime: enforceTime, } - c.chain = newCommitteeChain(c.db, &config, signerThreshold, enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) }) + c.chain = NewTestCommitteeChain(c.db, &config, signerThreshold, enforceTime, c.clock) return c } func (c *committeeChainTest) reloadChain() { - c.chain = newCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) }) + c.chain = NewTestCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, c.clock) } func (c *committeeChainTest) setClockPeriod(period float64) { diff --git a/beacon/light/head_tracker.go b/beacon/light/head_tracker.go new file mode 100644 index 000000000000..579e1b53daa9 --- /dev/null +++ b/beacon/light/head_tracker.go @@ -0,0 +1,150 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package light + +import ( + "errors" + "sync" + "time" + + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/log" +) + +// HeadTracker keeps track of the latest validated head and the "prefetch" head +// which is the (not necessarily validated) head announced by the majority of +// servers. +type HeadTracker struct { + lock sync.RWMutex + committeeChain *CommitteeChain + minSignerCount int + signedHead types.SignedHeader + hasSignedHead bool + finalityUpdate types.FinalityUpdate + hasFinalityUpdate bool + prefetchHead types.HeadInfo + changeCounter uint64 +} + +// NewHeadTracker creates a new HeadTracker. +func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int) *HeadTracker { + return &HeadTracker{ + committeeChain: committeeChain, + minSignerCount: minSignerCount, + } +} + +// ValidatedHead returns the latest validated head. +func (h *HeadTracker) ValidatedHead() (types.SignedHeader, bool) { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.signedHead, h.hasSignedHead +} + +// ValidatedHead returns the latest validated head. +func (h *HeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.finalityUpdate, h.hasFinalityUpdate +} + +// Validate validates the given signed head. If the head is successfully validated +// and it is better than the old validated head (higher slot or same slot and more +// signers) then ValidatedHead is updated. The boolean return flag signals if +// ValidatedHead has been changed. +func (h *HeadTracker) ValidateHead(head types.SignedHeader) (bool, error) { + h.lock.Lock() + defer h.lock.Unlock() + + replace, err := h.validate(head, h.signedHead) + if replace { + h.signedHead, h.hasSignedHead = head, true + h.changeCounter++ + } + return replace, err +} + +func (h *HeadTracker) ValidateFinality(update types.FinalityUpdate) (bool, error) { + h.lock.Lock() + defer h.lock.Unlock() + + replace, err := h.validate(update.SignedHeader(), h.finalityUpdate.SignedHeader()) + if replace { + h.finalityUpdate, h.hasFinalityUpdate = update, true + h.changeCounter++ + } + return replace, err +} + +func (h *HeadTracker) validate(head, oldHead types.SignedHeader) (bool, error) { + signerCount := head.Signature.SignerCount() + if signerCount < h.minSignerCount { + return false, errors.New("low signer count") + } + if head.Header.Slot < oldHead.Header.Slot || (head.Header.Slot == oldHead.Header.Slot && signerCount <= oldHead.Signature.SignerCount()) { + return false, nil + } + sigOk, age, err := h.committeeChain.VerifySignedHeader(head) + if err != nil { + return false, err + } + if age < 0 { + log.Warn("Future signed head received", "age", age) + } + if age > time.Minute*2 { + log.Warn("Old signed head received", "age", age) + } + if !sigOk { + return false, errors.New("invalid header signature") + } + return true, nil +} + +// PrefetchHead returns the latest known prefetch head's head info. +// This head can be used to start fetching related data hoping that it will be +// validated soon. +// Note that the prefetch head cannot be validated cryptographically so it should +// only be used as a performance optimization hint. +func (h *HeadTracker) PrefetchHead() types.HeadInfo { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.prefetchHead +} + +// SetPrefetchHead sets the prefetch head info. +// Note that HeadTracker does not verify the prefetch head, just acts as a thread +// safe bulletin board. +func (h *HeadTracker) SetPrefetchHead(head types.HeadInfo) { + h.lock.Lock() + defer h.lock.Unlock() + + if head == h.prefetchHead { + return + } + h.prefetchHead = head + h.changeCounter++ +} + +func (h *HeadTracker) ChangeCounter() uint64 { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.changeCounter +} diff --git a/beacon/light/request/scheduler.go b/beacon/light/request/scheduler.go new file mode 100644 index 000000000000..20f811900ecf --- /dev/null +++ b/beacon/light/request/scheduler.go @@ -0,0 +1,401 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package request + +import ( + "sync" + + "github.com/ethereum/go-ethereum/log" +) + +// Module represents a mechanism which is typically responsible for downloading +// and updating a passive data structure. It does not directly interact with the +// servers. It can start requests using the Requester interface, maintain its +// internal state by receiving and processing Events and update its target data +// structure based on the obtained data. +// It is the Scheduler's responsibility to feed events to the modules, call +// Process as long as there might be something to process and then generate request +// candidates using MakeRequest and start the best possible requests. +// Modules are called by Scheduler whenever a global trigger is fired. All events +// fire the trigger. Changing a target data structure also triggers a next +// processing round as it could make further actions possible either by the same +// or another Module. +type Module interface { + // Process is a non-blocking function responsible for starting requests, + // processing events and updating the target data structures(s) and the + // internal state of the module. Module state typically consists of information + // about pending requests and registered servers. + // Process is always called after an event is received or after a target data + // structure has been changed. + // + // Note: Process functions of different modules are never called concurrently; + // they are called by Scheduler in the same order of priority as they were + // registered in. + Process(Requester, []Event) +} + +// Requester allows Modules to obtain the list of momentarily available servers, +// start new requests and report server failure when a response has been proven +// to be invalid in the processing phase. +// Note that all Requester functions should be safe to call from Module.Process. +type Requester interface { + CanSendTo() []Server + Send(Server, Request) ID + Fail(Server, string) +} + +// Scheduler is a modular network data retrieval framework that coordinates multiple +// servers and retrieval mechanisms (modules). It implements a trigger mechanism +// that calls the Process function of registered modules whenever either the state +// of existing data structures or events coming from registered servers could +// allow new operations. +type Scheduler struct { + lock sync.Mutex + modules []Module // first has highest priority + names map[Module]string + servers map[server]struct{} + targets map[targetData]uint64 + + requesterLock sync.RWMutex + serverOrder []server + pending map[ServerAndID]pendingRequest + + // eventLock guards access to the events list. Note that eventLock can be + // locked either while lock is locked or unlocked but lock cannot be locked + // while eventLock is locked. + eventLock sync.Mutex + events []Event + stopCh chan chan struct{} + + triggerCh chan struct{} // restarts waiting sync loop + // if trigger has already been fired then send to testWaitCh blocks until + // the triggered processing round is finished + testWaitCh chan struct{} +} + +type ( + // Server identifies a server without allowing any direct interaction. + // Note: server interface is used by Scheduler and Tracker but not used by + // the modules that do not interact with them directly. + // In order to make module testing easier, Server interface is used in + // events and modules. + Server any + Request any + Response any + ID uint64 + ServerAndID struct { + Server Server + ID ID + } +) + +// targetData represents a registered target data structure that increases its +// ChangeCounter whenever it has been changed. +type targetData interface { + ChangeCounter() uint64 +} + +// pendingRequest keeps track of sent and not yet finalized requests and their +// sender modules. +type pendingRequest struct { + request Request + module Module +} + +// NewScheduler creates a new Scheduler. +func NewScheduler() *Scheduler { + s := &Scheduler{ + servers: make(map[server]struct{}), + names: make(map[Module]string), + pending: make(map[ServerAndID]pendingRequest), + targets: make(map[targetData]uint64), + stopCh: make(chan chan struct{}), + // Note: testWaitCh should not have capacity in order to ensure + // that after a trigger happens testWaitCh will block until the resulting + // processing round has been finished + triggerCh: make(chan struct{}, 1), + testWaitCh: make(chan struct{}), + } + return s +} + +// RegisterTarget registers a target data structure, ensuring that any changes +// made to it trigger a new round of Module.Process calls, giving a chance to +// modules to react to the changes. +func (s *Scheduler) RegisterTarget(t targetData) { + s.lock.Lock() + defer s.lock.Unlock() + + s.targets[t] = 0 +} + +// RegisterModule registers a module. Should be called before starting the scheduler. +// In each processing round the order of module processing depends on the order of +// registration. +func (s *Scheduler) RegisterModule(m Module, name string) { + s.lock.Lock() + defer s.lock.Unlock() + + s.modules = append(s.modules, m) + s.names[m] = name +} + +// RegisterServer registers a new server. +func (s *Scheduler) RegisterServer(server server) { + s.lock.Lock() + defer s.lock.Unlock() + + s.addEvent(Event{Type: EvRegistered, Server: server}) + server.subscribe(func(event Event) { + event.Server = server + s.addEvent(event) + }) +} + +// UnregisterServer removes a registered server. +func (s *Scheduler) UnregisterServer(server server) { + s.lock.Lock() + defer s.lock.Unlock() + + server.unsubscribe() + s.addEvent(Event{Type: EvUnregistered, Server: server}) +} + +// Start starts the scheduler. It should be called after registering all modules +// and before registering any servers. +func (s *Scheduler) Start() { + go s.syncLoop() +} + +// Stop stops the scheduler. +func (s *Scheduler) Stop() { + stop := make(chan struct{}) + s.stopCh <- stop + <-stop + s.lock.Lock() + for server := range s.servers { + server.unsubscribe() + } + s.servers = nil + s.lock.Unlock() +} + +// syncLoop is the main event loop responsible for event/data processing and +// sending new requests. +// A round of processing starts whenever the global trigger is fired. Triggers +// fired during a processing round ensure that there is going to be a next round. +func (s *Scheduler) syncLoop() { + for { + s.lock.Lock() + s.processRound() + s.lock.Unlock() + loop: + for { + select { + case stop := <-s.stopCh: + close(stop) + return + case <-s.triggerCh: + break loop + case <-s.testWaitCh: + } + } + } +} + +// targetChanged returns true if a registered target data structure has been +// changed since the last call to this function. +func (s *Scheduler) targetChanged() (changed bool) { + for target, counter := range s.targets { + if newCounter := target.ChangeCounter(); newCounter != counter { + s.targets[target] = newCounter + changed = true + } + } + return +} + +// processRound runs an entire processing round. It calls the Process functions +// of all modules, passing all relevant events and repeating Process calls as +// long as any changes have been made to the registered target data structures. +// Once all events have been processed and a stable state has been achieved, +// requests are generated and sent if necessary and possible. +func (s *Scheduler) processRound() { + for { + log.Trace("Processing modules") + filteredEvents := s.filterEvents() + for _, module := range s.modules { + log.Trace("Processing module", "name", s.names[module], "events", len(filteredEvents[module])) + module.Process(requester{s, module}, filteredEvents[module]) + } + if !s.targetChanged() { + break + } + } +} + +// Trigger starts a new processing round. If fired during processing, it ensures +// another full round of processing all modules. +func (s *Scheduler) Trigger() { + select { + case s.triggerCh <- struct{}{}: + default: + } +} + +// addEvent adds an event to be processed in the next round. Note that it can be +// called regardless of the state of the lock mutex, making it safe for use in +// the server event callback. +func (s *Scheduler) addEvent(event Event) { + s.eventLock.Lock() + s.events = append(s.events, event) + s.eventLock.Unlock() + s.Trigger() +} + +// filterEvent sorts each Event either as a request event or a server event, +// depending on its type. Request events are also sorted in a map based on the +// module that originally initiated the request. It also ensures that no events +// related to a server are returned before EvRegistered or after EvUnregistered. +// In case of an EvUnregistered server event it also closes all pending requests +// to the given server by adding a failed request event (EvFail), ensuring that +// all requests get finalized and thereby allowing the module logic to be safe +// and simple. +func (s *Scheduler) filterEvents() map[Module][]Event { + s.eventLock.Lock() + events := s.events + s.events = nil + s.eventLock.Unlock() + + s.requesterLock.Lock() + defer s.requesterLock.Unlock() + + filteredEvents := make(map[Module][]Event) + for _, event := range events { + server := event.Server.(server) + if _, ok := s.servers[server]; !ok && event.Type != EvRegistered { + continue // before EvRegister or after EvUnregister, discard + } + + if event.IsRequestEvent() { + sid, _, _ := event.RequestInfo() + pending, ok := s.pending[sid] + if !ok { + continue // request already closed, ignore further events + } + if event.Type == EvResponse || event.Type == EvFail { + delete(s.pending, sid) // final event, close pending request + } + filteredEvents[pending.module] = append(filteredEvents[pending.module], event) + } else { + switch event.Type { + case EvRegistered: + s.servers[server] = struct{}{} + s.serverOrder = append(s.serverOrder, nil) + copy(s.serverOrder[1:], s.serverOrder[:len(s.serverOrder)-1]) + s.serverOrder[0] = server + case EvUnregistered: + s.closePending(event.Server, filteredEvents) + delete(s.servers, server) + for i, srv := range s.serverOrder { + if srv == server { + copy(s.serverOrder[i:len(s.serverOrder)-1], s.serverOrder[i+1:]) + s.serverOrder = s.serverOrder[:len(s.serverOrder)-1] + break + } + } + } + for _, module := range s.modules { + filteredEvents[module] = append(filteredEvents[module], event) + } + } + } + return filteredEvents +} + +// closePending closes all pending requests to the given server and adds an EvFail +// event to properly finalize them +func (s *Scheduler) closePending(server Server, filteredEvents map[Module][]Event) { + for sid, pending := range s.pending { + if sid.Server == server { + filteredEvents[pending.module] = append(filteredEvents[pending.module], Event{ + Type: EvFail, + Server: server, + Data: RequestResponse{ + ID: sid.ID, + Request: pending.request, + }, + }) + delete(s.pending, sid) + } + } +} + +// requester implements Requester. Note that while requester basically wraps +// Scheduler (with the added information of the currently processed Module), all +// functions are safe to call from Module.Process which is running while +// the Scheduler.lock mutex is held. +type requester struct { + *Scheduler + module Module +} + +// CanSendTo returns the list of currently available servers. It also returns +// them in an order of least to most recently used, ensuring a round-robin usage +// of suitable servers if the module always chooses the first suitable one. +func (s requester) CanSendTo() []Server { + s.requesterLock.RLock() + defer s.requesterLock.RUnlock() + + list := make([]Server, 0, len(s.serverOrder)) + for _, server := range s.serverOrder { + if server.canRequestNow() { + list = append(list, server) + } + } + return list +} + +// Send sends a request and adds an entry to Scheduler.pending map, ensuring that +// related request events will be delivered to the sender Module. +func (s requester) Send(srv Server, req Request) ID { + s.requesterLock.Lock() + defer s.requesterLock.Unlock() + + server := srv.(server) + id := server.sendRequest(req) + sid := ServerAndID{Server: srv, ID: id} + s.pending[sid] = pendingRequest{request: req, module: s.module} + for i, ss := range s.serverOrder { + if ss == server { + copy(s.serverOrder[i:len(s.serverOrder)-1], s.serverOrder[i+1:]) + s.serverOrder[len(s.serverOrder)-1] = server + return id + } + } + log.Error("Target server not found in ordered list of registered servers") + return id +} + +// Fail should be called when a server delivers invalid or useless information. +// Calling Fail disables the given server for a period that is initially short +// but is exponentially growing if it happens frequently. This results in a +// somewhat fault tolerant operation that avoids hammering servers with requests +// that they cannot serve but still gives them a chance periodically. +func (s requester) Fail(srv Server, desc string) { + srv.(server).fail(desc) +} diff --git a/beacon/light/request/scheduler_test.go b/beacon/light/request/scheduler_test.go new file mode 100644 index 000000000000..7d5a56707864 --- /dev/null +++ b/beacon/light/request/scheduler_test.go @@ -0,0 +1,122 @@ +package request + +import ( + "reflect" + "testing" +) + +func TestEventFilter(t *testing.T) { + s := NewScheduler() + module1 := &testModule{name: "module1"} + module2 := &testModule{name: "module2"} + s.RegisterModule(module1, "module1") + s.RegisterModule(module2, "module2") + s.Start() + // startup process round without events + s.testWaitCh <- struct{}{} + module1.expProcess(t, nil) + module2.expProcess(t, nil) + srv := &testServer{} + // register server; both modules should receive server event + s.RegisterServer(srv) + s.testWaitCh <- struct{}{} + module1.expProcess(t, []Event{ + {Type: EvRegistered, Server: srv}, + }) + module2.expProcess(t, []Event{ + {Type: EvRegistered, Server: srv}, + }) + // let module1 send a request + srv.canRequest = 1 + module1.sendReq = testRequest + s.Trigger() + // in first triggered round module1 sends the request, no events yet + s.testWaitCh <- struct{}{} + module1.expProcess(t, nil) + module2.expProcess(t, nil) + // server emits EvTimeout; only module1 should receive it + srv.eventCb(Event{Type: EvTimeout, Data: RequestResponse{ID: 1, Request: testRequest}}) + s.testWaitCh <- struct{}{} + module1.expProcess(t, []Event{ + {Type: EvTimeout, Server: srv, Data: RequestResponse{ID: 1, Request: testRequest}}, + }) + module2.expProcess(t, nil) + // unregister server; both modules should receive server event + s.UnregisterServer(srv) + s.testWaitCh <- struct{}{} + module1.expProcess(t, []Event{ + // module1 should also receive EvFail on its pending request + {Type: EvFail, Server: srv, Data: RequestResponse{ID: 1, Request: testRequest}}, + {Type: EvUnregistered, Server: srv}, + }) + module2.expProcess(t, []Event{ + {Type: EvUnregistered, Server: srv}, + }) + // response after server unregistered; should be discarded + srv.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + s.testWaitCh <- struct{}{} + module1.expProcess(t, nil) + module2.expProcess(t, nil) + // no more process rounds expected; shut down + s.testWaitCh <- struct{}{} + module1.expNoMoreProcess(t) + module2.expNoMoreProcess(t) + s.Stop() +} + +type testServer struct { + eventCb func(Event) + lastID ID + canRequest int +} + +func (s *testServer) subscribe(eventCb func(Event)) { + s.eventCb = eventCb +} + +func (s *testServer) canRequestNow() bool { + return s.canRequest > 0 +} + +func (s *testServer) sendRequest(req Request) ID { + s.canRequest-- + s.lastID++ + return s.lastID +} + +func (s *testServer) fail(string) {} +func (s *testServer) unsubscribe() {} + +type testModule struct { + name string + processed [][]Event + sendReq Request +} + +func (m *testModule) Process(requester Requester, events []Event) { + m.processed = append(m.processed, events) + if m.sendReq != nil { + if cs := requester.CanSendTo(); len(cs) > 0 { + requester.Send(cs[0], m.sendReq) + } + } +} + +func (m *testModule) expProcess(t *testing.T, expEvents []Event) { + if len(m.processed) == 0 { + t.Errorf("Missing call to %s.Process", m.name) + return + } + events := m.processed[0] + m.processed = m.processed[1:] + if !reflect.DeepEqual(events, expEvents) { + t.Errorf("Call to %s.Process with wrong events (expected %v, got %v)", m.name, expEvents, events) + } +} + +func (m *testModule) expNoMoreProcess(t *testing.T) { + for len(m.processed) > 0 { + t.Errorf("Unexpected call to %s.Process with events %v", m.name, m.processed[0]) + m.processed = m.processed[1:] + } +} diff --git a/beacon/light/request/server.go b/beacon/light/request/server.go new file mode 100644 index 000000000000..407eb69f497e --- /dev/null +++ b/beacon/light/request/server.go @@ -0,0 +1,439 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package request + +import ( + "math" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/log" +) + +var ( + // request events + EvResponse = &EventType{Name: "response", requestEvent: true} // data: RequestResponse; sent by requestServer + EvFail = &EventType{Name: "fail", requestEvent: true} // data: RequestResponse; sent by requestServer + EvTimeout = &EventType{Name: "timeout", requestEvent: true} // data: RequestResponse; sent by serverWithTimeout + // server events + EvRegistered = &EventType{Name: "registered"} // data: nil; sent by Scheduler + EvUnregistered = &EventType{Name: "unregistered"} // data: nil; sent by Scheduler + EvCanRequestAgain = &EventType{Name: "canRequestAgain"} // data: nil; sent by serverWithLimits +) + +const ( + softRequestTimeout = time.Second // allow resending request to a different server but do not cancel yet + hardRequestTimeout = time.Second * 10 // cancel request +) + +const ( + // serverWithLimits parameters + parallelAdjustUp = 0.1 // adjust parallelLimit up in case of success under full load + parallelAdjustDown = 1 // adjust parallelLimit down in case of timeout/failure + minParallelLimit = 1 // parallelLimit lower bound + defaultParallelLimit = 3 // parallelLimit initial value + minFailureDelay = time.Millisecond * 100 // minimum disable time in case of request failure + maxFailureDelay = time.Minute // maximum disable time in case of request failure + maxServerEventBuffer = 5 // server event allowance buffer limit + maxServerEventRate = time.Second // server event allowance buffer recharge rate +) + +// requestServer can send requests in a non-blocking way and feed back events +// through the event callback. After each request it should send back either +// EvResponse or EvFail. Additionally, it may also send application-defined +// events that the Modules can interpret. +type requestServer interface { + Subscribe(eventCallback func(Event)) + SendRequest(ID, Request) + Unsubscribe() +} + +// server is implemented by a requestServer wrapped into serverWithTimeout and +// serverWithLimits and is used by Scheduler. +// In addition to requestServer functionality, server can also handle timeouts, +// limit the number of parallel in-flight requests and temporarily disable +// new requests based on timeouts and response failures. +type server interface { + subscribe(eventCallback func(Event)) + canRequestNow() bool + sendRequest(Request) ID + fail(string) + unsubscribe() +} + +// NewServer wraps a requestServer and returns a server +func NewServer(rs requestServer, clock mclock.Clock) server { + s := &serverWithLimits{} + s.parent = rs + s.serverWithTimeout.init(clock) + s.init() + return s +} + +// EventType identifies an event type, either related to a request or the server +// in general. Server events can also be externally defined. +type EventType struct { + Name string + requestEvent bool // all request events are pre-defined in request package +} + +// Event describes an event where the type of Data depends on Type. +// Server field is not required when sent through the event callback; it is filled +// out when processed by the Scheduler. Note that the Scheduler can also create +// and send events (EvRegistered, EvUnregistered) directly. +type Event struct { + Type *EventType + Server Server // filled by Scheduler + Data any +} + +// IsRequestEvent returns true if the event is a request event +func (e *Event) IsRequestEvent() bool { + return e.Type.requestEvent +} + +// RequestInfo assumes that the event is a request event and returns its contents +// in a convenient form. +func (e *Event) RequestInfo() (ServerAndID, Request, Response) { + data := e.Data.(RequestResponse) + return ServerAndID{Server: e.Server, ID: data.ID}, data.Request, data.Response +} + +// RequestResponse is the Data type of request events. +type RequestResponse struct { + ID ID + Request Request + Response Response +} + +// serverWithTimeout wraps a requestServer and introduces timeouts. +// The request's lifecycle is concluded if EvResponse or EvFail emitted by the +// parent requestServer. If this does not happen until softRequestTimeout then +// EvTimeout is emitted, after which the final EvResponse or EvFail is still +// guaranteed to follow. +// If the parent fails to send this final event for hardRequestTimeout then +// serverWithTimeout emits EvFail and discards any further events from the +// parent related to the given request. +type serverWithTimeout struct { + parent requestServer + lock sync.Mutex + clock mclock.Clock + childEventCb func(event Event) + timeouts map[ID]mclock.Timer + lastID ID +} + +// init initializes serverWithTimeout +func (s *serverWithTimeout) init(clock mclock.Clock) { + s.clock = clock + s.timeouts = make(map[ID]mclock.Timer) +} + +// subscribe subscribes to events which include parent (requestServer) events +// plus EvTimeout. +func (s *serverWithTimeout) subscribe(eventCallback func(event Event)) { + s.lock.Lock() + defer s.lock.Unlock() + + s.childEventCb = eventCallback + s.parent.Subscribe(s.eventCallback) +} + +// sendRequest generated a new request ID, emits EvRequest, sets up the timeout +// timer, then sends the request through the parent (requestServer). +func (s *serverWithTimeout) sendRequest(request Request) (reqId ID) { + s.lock.Lock() + s.lastID++ + id := s.lastID + s.startTimeout(RequestResponse{ID: id, Request: request}) + s.lock.Unlock() + s.parent.SendRequest(id, request) + return id +} + +// eventCallback is called by parent (requestServer) event subscription. +func (s *serverWithTimeout) eventCallback(event Event) { + s.lock.Lock() + defer s.lock.Unlock() + + switch event.Type { + case EvResponse, EvFail: + id := event.Data.(RequestResponse).ID + if timer, ok := s.timeouts[id]; ok { + // Note: if stopping the timer is unsuccessful then the resulting AfterFunc + // call will just do nothing + timer.Stop() + delete(s.timeouts, id) + s.childEventCb(event) + } + default: + s.childEventCb(event) + } +} + +// startTimeout starts a timeout timer for the given request. +func (s *serverWithTimeout) startTimeout(reqData RequestResponse) { + id := reqData.ID + s.timeouts[id] = s.clock.AfterFunc(softRequestTimeout, func() { + s.lock.Lock() + if _, ok := s.timeouts[id]; !ok { + s.lock.Unlock() + return + } + s.timeouts[id] = s.clock.AfterFunc(hardRequestTimeout-softRequestTimeout, func() { + s.lock.Lock() + if _, ok := s.timeouts[id]; !ok { + s.lock.Unlock() + return + } + delete(s.timeouts, id) + childEventCb := s.childEventCb + s.lock.Unlock() + childEventCb(Event{Type: EvFail, Data: reqData}) + }) + childEventCb := s.childEventCb + s.lock.Unlock() + childEventCb(Event{Type: EvTimeout, Data: reqData}) + }) +} + +// stop stops all goroutines associated with the server. +func (s *serverWithTimeout) unsubscribe() { + s.lock.Lock() + defer s.lock.Unlock() + + for _, timer := range s.timeouts { + if timer != nil { + timer.Stop() + } + } + s.childEventCb = nil + s.parent.Unsubscribe() +} + +// serverWithLimits wraps serverWithTimeout and implements server. It limits the +// number of parallel in-flight requests and prevents sending new requests when a +// pending one has already timed out. Server events are also rate limited. +// It also implements a failure delay mechanism that adds an exponentially growing +// delay each time a request fails (wrong answer or hard timeout). This makes the +// syncing mechanism less brittle as temporary failures of the server might happen +// sometimes, but still avoids hammering a non-functional server with requests. +type serverWithLimits struct { + serverWithTimeout + lock sync.Mutex + childEventCb func(event Event) + softTimeouts map[ID]struct{} + pendingCount, timeoutCount int + parallelLimit float32 + sendEvent bool + delayTimer mclock.Timer + delayCounter int + failureDelayEnd mclock.AbsTime + failureDelay float64 + serverEventBuffer int + eventBufferUpdated mclock.AbsTime +} + +// init initializes serverWithLimits +func (s *serverWithLimits) init() { + s.softTimeouts = make(map[ID]struct{}) + s.parallelLimit = defaultParallelLimit + s.serverEventBuffer = maxServerEventBuffer +} + +// subscribe subscribes to events which include parent (serverWithTimeout) events +// plus EvCanRequestAgain. +func (s *serverWithLimits) subscribe(eventCallback func(event Event)) { + s.lock.Lock() + defer s.lock.Unlock() + + s.childEventCb = eventCallback + s.serverWithTimeout.subscribe(s.eventCallback) +} + +// eventCallback is called by parent (serverWithTimeout) event subscription. +func (s *serverWithLimits) eventCallback(event Event) { + s.lock.Lock() + var sendCanRequestAgain bool + passEvent := true + switch event.Type { + case EvTimeout: + id := event.Data.(RequestResponse).ID + s.softTimeouts[id] = struct{}{} + s.timeoutCount++ + s.parallelLimit -= parallelAdjustDown + if s.parallelLimit < minParallelLimit { + s.parallelLimit = minParallelLimit + } + log.Debug("Server timeout", "count", s.timeoutCount, "parallelLimit", s.parallelLimit) + case EvResponse, EvFail: + id := event.Data.(RequestResponse).ID + if _, ok := s.softTimeouts[id]; ok { + delete(s.softTimeouts, id) + s.timeoutCount-- + log.Debug("Server timeout finalized", "count", s.timeoutCount, "parallelLimit", s.parallelLimit) + } + if event.Type == EvResponse && s.pendingCount >= int(s.parallelLimit) { + s.parallelLimit += parallelAdjustUp + } + s.pendingCount-- + if s.canRequest() { + sendCanRequestAgain = s.sendEvent + s.sendEvent = false + } + if event.Type == EvFail { + s.failLocked("failed request") + } + default: + // server event; check rate limit + if s.serverEventBuffer < maxServerEventBuffer { + now := s.clock.Now() + sinceUpdate := time.Duration(now - s.eventBufferUpdated) + if sinceUpdate >= maxServerEventRate*time.Duration(maxServerEventBuffer-s.serverEventBuffer) { + s.serverEventBuffer = maxServerEventBuffer + s.eventBufferUpdated = now + } else { + addBuffer := int(sinceUpdate / maxServerEventRate) + s.serverEventBuffer += addBuffer + s.eventBufferUpdated += mclock.AbsTime(maxServerEventRate * time.Duration(addBuffer)) + } + } + if s.serverEventBuffer > 0 { + s.serverEventBuffer-- + } else { + passEvent = false + } + } + childEventCb := s.childEventCb + s.lock.Unlock() + if passEvent { + childEventCb(event) + } + if sendCanRequestAgain { + childEventCb(Event{Type: EvCanRequestAgain}) + } +} + +// sendRequest sends a request through the parent (serverWithTimeout). +func (s *serverWithLimits) sendRequest(request Request) (reqId ID) { + s.lock.Lock() + s.pendingCount++ + s.lock.Unlock() + return s.serverWithTimeout.sendRequest(request) +} + +// stop stops all goroutines associated with the server. +func (s *serverWithLimits) unsubscribe() { + s.lock.Lock() + defer s.lock.Unlock() + + if s.delayTimer != nil { + s.delayTimer.Stop() + s.delayTimer = nil + } + s.childEventCb = nil + s.serverWithTimeout.unsubscribe() +} + +// canRequest checks whether a new request can be started. +func (s *serverWithLimits) canRequest() bool { + if s.delayTimer != nil || s.pendingCount >= int(s.parallelLimit) || s.timeoutCount > 0 { + return false + } + if s.parallelLimit < minParallelLimit { + s.parallelLimit = minParallelLimit + } + return true +} + +// canRequestNow checks whether a new request can be started, according to the +// current in-flight request count and parallelLimit, and also the failure delay +// timer. +// If it returns false then it is guaranteed that an EvCanRequestAgain will be +// sent whenever the server becomes available for requesting again. +func (s *serverWithLimits) canRequestNow() bool { + var sendCanRequestAgain bool + s.lock.Lock() + canRequest := s.canRequest() + if canRequest { + sendCanRequestAgain = s.sendEvent + s.sendEvent = false + } + childEventCb := s.childEventCb + s.lock.Unlock() + if sendCanRequestAgain { + childEventCb(Event{Type: EvCanRequestAgain}) + } + return canRequest +} + +// delay sets the delay timer to the given duration, disabling new requests for +// the given period. +func (s *serverWithLimits) delay(delay time.Duration) { + if s.delayTimer != nil { + // Note: if stopping the timer is unsuccessful then the resulting AfterFunc + // call will just do nothing + s.delayTimer.Stop() + s.delayTimer = nil + } + + s.delayCounter++ + delayCounter := s.delayCounter + log.Debug("Server delay started", "length", delay) + s.delayTimer = s.clock.AfterFunc(delay, func() { + log.Debug("Server delay ended", "length", delay) + var sendCanRequestAgain bool + s.lock.Lock() + if s.delayTimer != nil && s.delayCounter == delayCounter { // do nothing if there is a new timer now + s.delayTimer = nil + if s.canRequest() { + sendCanRequestAgain = s.sendEvent + s.sendEvent = false + } + } + childEventCb := s.childEventCb + s.lock.Unlock() + if sendCanRequestAgain { + childEventCb(Event{Type: EvCanRequestAgain}) + } + }) +} + +// fail reports that a response from the server was found invalid by the processing +// Module, disabling new requests for a dynamically adjusted time period. +func (s *serverWithLimits) fail(desc string) { + s.lock.Lock() + defer s.lock.Unlock() + + s.failLocked(desc) +} + +// failLocked calculates the dynamic failure delay and applies it. +func (s *serverWithLimits) failLocked(desc string) { + log.Debug("Server error", "description", desc) + s.failureDelay *= 2 + now := s.clock.Now() + if now > s.failureDelayEnd { + s.failureDelay *= math.Pow(2, -float64(now-s.failureDelayEnd)/float64(maxFailureDelay)) + } + if s.failureDelay < float64(minFailureDelay) { + s.failureDelay = float64(minFailureDelay) + } + s.failureDelayEnd = now + mclock.AbsTime(s.failureDelay) + s.delay(time.Duration(s.failureDelay)) +} diff --git a/beacon/light/request/server_test.go b/beacon/light/request/server_test.go new file mode 100644 index 000000000000..b6b9edf9a056 --- /dev/null +++ b/beacon/light/request/server_test.go @@ -0,0 +1,158 @@ +package request + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +const ( + testRequest = "Life, the Universe, and Everything" + testResponse = 42 +) + +var testEventType = &EventType{Name: "testEvent"} + +func TestServerEvents(t *testing.T) { + rs := &testRequestServer{} + clock := &mclock.Simulated{} + srv := NewServer(rs, clock) + var lastEventType *EventType + srv.subscribe(func(event Event) { lastEventType = event.Type }) + evTypeName := func(evType *EventType) string { + if evType == nil { + return "none" + } + return evType.Name + } + expEvent := func(expType *EventType) { + if lastEventType != expType { + t.Errorf("Wrong event type (expected %s, got %s)", evTypeName(expType), evTypeName(lastEventType)) + } + lastEventType = nil + } + // user events should simply be passed through + rs.eventCb(Event{Type: testEventType}) + expEvent(testEventType) + // send request, soft timeout, then valid response + srv.sendRequest(testRequest) + clock.WaitForTimers(1) + clock.Run(softRequestTimeout) + expEvent(EvTimeout) + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expEvent(EvResponse) + // send request, hard timeout (response after hard timeout should be ignored) + srv.sendRequest(testRequest) + clock.WaitForTimers(1) + clock.Run(softRequestTimeout) + expEvent(EvTimeout) + clock.WaitForTimers(1) + clock.Run(hardRequestTimeout) + expEvent(EvFail) + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expEvent(nil) +} + +func TestServerParallel(t *testing.T) { + rs := &testRequestServer{} + srv := NewServer(rs, &mclock.Simulated{}) + srv.subscribe(func(event Event) {}) + + expSend := func(expSent int) { + var sent int + for sent <= expSent { + if !srv.canRequestNow() { + break + } + sent++ + srv.sendRequest(testRequest) + } + if sent != expSent { + t.Errorf("Wrong number of parallel requests accepted (expected %d, got %d)", expSent, sent) + } + } + // max out parallel allowance + expSend(defaultParallelLimit) + // 1 answered, should accept 1 more + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expSend(1) + // 2 answered, should accept 2 more + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 2, Request: testRequest, Response: testResponse}}) + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 3, Request: testRequest, Response: testResponse}}) + expSend(2) + // failed request, should decrease allowance and not accept more + rs.eventCb(Event{Type: EvFail, Data: RequestResponse{ID: 4, Request: testRequest}}) + expSend(0) + srv.unsubscribe() +} + +func TestServerFail(t *testing.T) { + rs := &testRequestServer{} + clock := &mclock.Simulated{} + srv := NewServer(rs, clock) + srv.subscribe(func(event Event) {}) + expCanRequest := func(expCanRequest bool) { + if canRequest := srv.canRequestNow(); canRequest != expCanRequest { + t.Errorf("Wrong result for canRequestNow (expected %v, got %v)", expCanRequest, canRequest) + } + } + // timed out request + expCanRequest(true) + srv.sendRequest(testRequest) + clock.WaitForTimers(1) + expCanRequest(true) + clock.Run(softRequestTimeout) + expCanRequest(false) // cannot request when there is a timed out request + rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}}) + expCanRequest(true) + // explicit server.Fail + srv.fail("") + clock.WaitForTimers(1) + expCanRequest(false) // cannot request for a while after a failure + clock.Run(minFailureDelay) + expCanRequest(true) + // request returned with EvFail + srv.sendRequest(testRequest) + rs.eventCb(Event{Type: EvFail, Data: RequestResponse{ID: 2, Request: testRequest}}) + clock.WaitForTimers(1) + expCanRequest(false) // EvFail should also start failure delay + clock.Run(minFailureDelay) + expCanRequest(false) // second failure delay is longer, should still be disabled + clock.Run(minFailureDelay) + expCanRequest(true) + srv.unsubscribe() +} + +func TestServerEventRateLimit(t *testing.T) { + rs := &testRequestServer{} + clock := &mclock.Simulated{} + srv := NewServer(rs, clock) + var eventCount int + srv.subscribe(func(event Event) { + if !event.IsRequestEvent() { + eventCount++ + } + }) + expEvents := func(send, expAllowed int) { + eventCount = 0 + for sent := 0; sent < send; sent++ { + rs.eventCb(Event{Type: testEventType}) + } + if eventCount != expAllowed { + t.Errorf("Wrong number of server events passing rate limitation (sent %d, expected %d, got %d)", send, expAllowed, eventCount) + } + } + expEvents(maxServerEventBuffer+5, maxServerEventBuffer) + clock.Run(maxServerEventRate) + expEvents(5, 1) + clock.Run(maxServerEventRate * maxServerEventBuffer * 2) + expEvents(maxServerEventBuffer+5, maxServerEventBuffer) +} + +type testRequestServer struct { + eventCb func(Event) +} + +func (rs *testRequestServer) Subscribe(eventCb func(Event)) { rs.eventCb = eventCb } +func (rs *testRequestServer) SendRequest(ID, Request) {} +func (rs *testRequestServer) Unsubscribe() {} diff --git a/beacon/light/sync/head_sync.go b/beacon/light/sync/head_sync.go new file mode 100644 index 000000000000..9fef95b0df79 --- /dev/null +++ b/beacon/light/sync/head_sync.go @@ -0,0 +1,176 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" +) + +type headTracker interface { + ValidateHead(head types.SignedHeader) (bool, error) + ValidateFinality(head types.FinalityUpdate) (bool, error) + SetPrefetchHead(head types.HeadInfo) +} + +// HeadSync implements request.Module; it updates the validated and prefetch +// heads of HeadTracker based on the EvHead and EvSignedHead events coming from +// registered servers. +// It can also postpone the validation of the latest announced signed head +// until the committee chain is synced up to at least the required period. +type HeadSync struct { + headTracker headTracker + chain committeeChain + nextSyncPeriod uint64 + chainInit bool + unvalidatedHeads map[request.Server]types.SignedHeader + unvalidatedFinality map[request.Server]types.FinalityUpdate + serverHeads map[request.Server]types.HeadInfo + headServerCount map[types.HeadInfo]headServerCount + headCounter uint64 + prefetchHead types.HeadInfo +} + +// headServerCount is associated with most recently seen head infos; it counts +// the number of servers currently having the given head info as their announced +// head and a counter signaling how recent that head is. +// This data is used for selecting the prefetch head. +type headServerCount struct { + serverCount int + headCounter uint64 +} + +// NewHeadSync creates a new HeadSync. +func NewHeadSync(headTracker headTracker, chain committeeChain) *HeadSync { + s := &HeadSync{ + headTracker: headTracker, + chain: chain, + unvalidatedHeads: make(map[request.Server]types.SignedHeader), + unvalidatedFinality: make(map[request.Server]types.FinalityUpdate), + serverHeads: make(map[request.Server]types.HeadInfo), + headServerCount: make(map[types.HeadInfo]headServerCount), + } + return s +} + +// Process implements request.Module. +func (s *HeadSync) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + switch event.Type { + case EvNewHead: + s.setServerHead(event.Server, event.Data.(types.HeadInfo)) + case EvNewSignedHead: + s.newSignedHead(event.Server, event.Data.(types.SignedHeader)) + case EvNewFinalityUpdate: + s.newFinalityUpdate(event.Server, event.Data.(types.FinalityUpdate)) + case request.EvUnregistered: + s.setServerHead(event.Server, types.HeadInfo{}) + delete(s.serverHeads, event.Server) + delete(s.unvalidatedHeads, event.Server) + } + } + + nextPeriod, chainInit := s.chain.NextSyncPeriod() + if nextPeriod != s.nextSyncPeriod || chainInit != s.chainInit { + s.nextSyncPeriod, s.chainInit = nextPeriod, chainInit + s.processUnvalidated() + } +} + +// newSignedHead handles received signed head; either validates it if the chain +// is properly synced or stores it for further validation. +func (s *HeadSync) newSignedHead(server request.Server, signedHead types.SignedHeader) { + if !s.chainInit || types.SyncPeriod(signedHead.SignatureSlot) > s.nextSyncPeriod { + s.unvalidatedHeads[server] = signedHead + return + } + s.headTracker.ValidateHead(signedHead) +} + +// newSignedHead handles received signed head; either validates it if the chain +// is properly synced or stores it for further validation. +func (s *HeadSync) newFinalityUpdate(server request.Server, finalityUpdate types.FinalityUpdate) { + if !s.chainInit || types.SyncPeriod(finalityUpdate.SignatureSlot) > s.nextSyncPeriod { + s.unvalidatedFinality[server] = finalityUpdate + return + } + s.headTracker.ValidateFinality(finalityUpdate) +} + +// processUnvalidatedHeads iterates the list of unvalidated heads and validates +// those which can be validated. +func (s *HeadSync) processUnvalidated() { + if !s.chainInit { + return + } + for server, signedHead := range s.unvalidatedHeads { + if types.SyncPeriod(signedHead.SignatureSlot) <= s.nextSyncPeriod { + s.headTracker.ValidateHead(signedHead) + delete(s.unvalidatedHeads, server) + } + } + for server, finalityUpdate := range s.unvalidatedFinality { + if types.SyncPeriod(finalityUpdate.SignatureSlot) <= s.nextSyncPeriod { + s.headTracker.ValidateFinality(finalityUpdate) + delete(s.unvalidatedFinality, server) + } + } +} + +// setServerHead processes non-validated server head announcements and updates +// the prefetch head if necessary. +func (s *HeadSync) setServerHead(server request.Server, head types.HeadInfo) bool { + if oldHead, ok := s.serverHeads[server]; ok { + if head == oldHead { + return false + } + h := s.headServerCount[oldHead] + if h.serverCount--; h.serverCount > 0 { + s.headServerCount[oldHead] = h + } else { + delete(s.headServerCount, oldHead) + } + } + if head != (types.HeadInfo{}) { + h, ok := s.headServerCount[head] + if !ok { + s.headCounter++ + h.headCounter = s.headCounter + } + h.serverCount++ + s.headServerCount[head] = h + s.serverHeads[server] = head + } else { + delete(s.serverHeads, server) + } + var ( + bestHead types.HeadInfo + bestHeadInfo headServerCount + ) + for head, headServerCount := range s.headServerCount { + if headServerCount.serverCount > bestHeadInfo.serverCount || + (headServerCount.serverCount == bestHeadInfo.serverCount && headServerCount.headCounter > bestHeadInfo.headCounter) { + bestHead, bestHeadInfo = head, headServerCount + } + } + if bestHead == s.prefetchHead { + return false + } + s.prefetchHead = bestHead + s.headTracker.SetPrefetchHead(bestHead) + return true +} diff --git a/beacon/light/sync/head_sync_test.go b/beacon/light/sync/head_sync_test.go new file mode 100644 index 000000000000..12faad62920e --- /dev/null +++ b/beacon/light/sync/head_sync_test.go @@ -0,0 +1,151 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "testing" + + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" +) + +var ( + testServer1 = "testServer1" + testServer2 = "testServer2" + testServer3 = "testServer3" + testServer4 = "testServer4" + + testHead0 = types.HeadInfo{} + testHead1 = types.HeadInfo{Slot: 123, BlockRoot: common.Hash{1}} + testHead2 = types.HeadInfo{Slot: 124, BlockRoot: common.Hash{2}} + testHead3 = types.HeadInfo{Slot: 124, BlockRoot: common.Hash{3}} + testHead4 = types.HeadInfo{Slot: 125, BlockRoot: common.Hash{4}} + + testSHead1 = types.SignedHeader{SignatureSlot: 0x0124, Header: types.Header{Slot: 0x0123, StateRoot: common.Hash{1}}} + testSHead2 = types.SignedHeader{SignatureSlot: 0x2010, Header: types.Header{Slot: 0x200e, StateRoot: common.Hash{2}}} + // testSHead3 is at the end of period 1 but signed in period 2 + testSHead3 = types.SignedHeader{SignatureSlot: 0x4000, Header: types.Header{Slot: 0x3fff, StateRoot: common.Hash{3}}} + testSHead4 = types.SignedHeader{SignatureSlot: 0x6444, Header: types.Header{Slot: 0x6443, StateRoot: common.Hash{4}}} +) + +func TestValidatedHead(t *testing.T) { + chain := &TestCommitteeChain{} + ht := &TestHeadTracker{} + headSync := NewHeadSync(ht, chain) + ts := NewTestScheduler(t, headSync) + + ht.ExpValidated(t, 0, nil) + + ts.AddServer(testServer1, 1) + ts.ServerEvent(EvNewSignedHead, testServer1, testSHead1) + ts.Run(1) + // announced head should be queued because of uninitialized chain + ht.ExpValidated(t, 1, nil) + + chain.SetNextSyncPeriod(0) // initialize chain + ts.Run(2) + // expect previously queued head to be validated + ht.ExpValidated(t, 2, []types.SignedHeader{testSHead1}) + + chain.SetNextSyncPeriod(1) + ts.ServerEvent(EvNewSignedHead, testServer1, testSHead2) + ts.AddServer(testServer2, 1) + ts.ServerEvent(EvNewSignedHead, testServer2, testSHead2) + ts.Run(3) + // expect both head announcements to be validated instantly + ht.ExpValidated(t, 3, []types.SignedHeader{testSHead2, testSHead2}) + + ts.ServerEvent(EvNewSignedHead, testServer1, testSHead3) + ts.AddServer(testServer3, 1) + ts.ServerEvent(EvNewSignedHead, testServer3, testSHead4) + ts.Run(4) + // future period annonced heads should be queued + ht.ExpValidated(t, 4, nil) + + chain.SetNextSyncPeriod(2) + ts.Run(5) + // testSHead3 can be validated now but not testSHead4 + ht.ExpValidated(t, 5, []types.SignedHeader{testSHead3}) + + // server 3 disconnected without proving period 3, its announced head should be dropped + ts.RemoveServer(testServer3) + ts.Run(6) + ht.ExpValidated(t, 6, nil) + + chain.SetNextSyncPeriod(3) + ts.Run(7) + // testSHead4 could be validated now but it's not queued by any registered server + ht.ExpValidated(t, 7, nil) + + ts.ServerEvent(EvNewSignedHead, testServer2, testSHead4) + ts.Run(8) + // now testSHead4 should be validated + ht.ExpValidated(t, 8, []types.SignedHeader{testSHead4}) +} + +func TestPrefetchHead(t *testing.T) { + chain := &TestCommitteeChain{} + ht := &TestHeadTracker{} + headSync := NewHeadSync(ht, chain) + ts := NewTestScheduler(t, headSync) + + ht.ExpPrefetch(t, 0, testHead0) // no servers registered + + ts.AddServer(testServer1, 1) + ts.ServerEvent(EvNewHead, testServer1, testHead1) + ts.Run(1) + ht.ExpPrefetch(t, 1, testHead1) // s1: h1 + + ts.AddServer(testServer2, 1) + ts.ServerEvent(EvNewHead, testServer2, testHead2) + ts.Run(2) + ht.ExpPrefetch(t, 2, testHead2) // s1: h1, s2: h2 + + ts.ServerEvent(EvNewHead, testServer1, testHead2) + ts.Run(3) + ht.ExpPrefetch(t, 3, testHead2) // s1: h2, s2: h2 + + ts.AddServer(testServer3, 1) + ts.ServerEvent(EvNewHead, testServer3, testHead3) + ts.Run(4) + ht.ExpPrefetch(t, 4, testHead2) // s1: h2, s2: h2, s3: h3 + + ts.AddServer(testServer4, 1) + ts.ServerEvent(EvNewHead, testServer4, testHead4) + ts.Run(5) + ht.ExpPrefetch(t, 5, testHead2) // s1: h2, s2: h2, s3: h3, s4: h4 + + ts.ServerEvent(EvNewHead, testServer2, testHead3) + ts.Run(6) + ht.ExpPrefetch(t, 6, testHead3) // s1: h2, s2: h3, s3: h3, s4: h4 + + ts.RemoveServer(testServer3) + ts.Run(7) + ht.ExpPrefetch(t, 7, testHead4) // s1: h2, s2: h3, s4: h4 + + ts.RemoveServer(testServer1) + ts.Run(8) + ht.ExpPrefetch(t, 8, testHead4) // s2: h3, s4: h4 + + ts.RemoveServer(testServer4) + ts.Run(9) + ht.ExpPrefetch(t, 9, testHead3) // s2: h3 + + ts.RemoveServer(testServer2) + ts.Run(10) + ht.ExpPrefetch(t, 10, testHead0) // no servers registered +} diff --git a/beacon/light/sync/test_helpers.go b/beacon/light/sync/test_helpers.go new file mode 100644 index 000000000000..a1ca2b590993 --- /dev/null +++ b/beacon/light/sync/test_helpers.go @@ -0,0 +1,254 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/beacon/light" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" +) + +type requestWithID struct { + sid request.ServerAndID + request request.Request +} + +type TestScheduler struct { + t *testing.T + module request.Module + events []request.Event + servers []request.Server + allowance map[request.Server]int + sent map[int][]requestWithID + testIndex int + expFail map[request.Server]int // expected Server.Fail calls during next Run + lastId request.ID +} + +func NewTestScheduler(t *testing.T, module request.Module) *TestScheduler { + return &TestScheduler{ + t: t, + module: module, + allowance: make(map[request.Server]int), + expFail: make(map[request.Server]int), + sent: make(map[int][]requestWithID), + } +} + +func (ts *TestScheduler) Run(testIndex int, exp ...any) { + expReqs := make([]requestWithID, len(exp)/2) + id := ts.lastId + for i := range expReqs { + id++ + expReqs[i] = requestWithID{ + sid: request.ServerAndID{Server: exp[i*2].(request.Server), ID: id}, + request: exp[i*2+1].(request.Request), + } + } + if len(expReqs) == 0 { + expReqs = nil + } + + ts.testIndex = testIndex + ts.module.Process(ts, ts.events) + ts.events = nil + + for server, count := range ts.expFail { + delete(ts.expFail, server) + if count == 0 { + continue + } + ts.t.Errorf("Missing %d Server.Fail(s) from server %s in test case #%d", count, server.(string), testIndex) + } + + if !reflect.DeepEqual(ts.sent[testIndex], expReqs) { + ts.t.Errorf("Wrong sent requests in test case #%d (expected %v, got %v)", testIndex, expReqs, ts.sent[testIndex]) + } +} + +func (ts *TestScheduler) CanSendTo() (cs []request.Server) { + for _, server := range ts.servers { + if ts.allowance[server] > 0 { + cs = append(cs, server) + } + } + return +} + +func (ts *TestScheduler) Send(server request.Server, req request.Request) request.ID { + ts.lastId++ + ts.sent[ts.testIndex] = append(ts.sent[ts.testIndex], requestWithID{ + sid: request.ServerAndID{Server: server, ID: ts.lastId}, + request: req, + }) + ts.allowance[server]-- + return ts.lastId +} + +func (ts *TestScheduler) Fail(server request.Server, desc string) { + if ts.expFail[server] == 0 { + ts.t.Errorf("Unexpected Fail from server %s in test case #%d: %s", server.(string), ts.testIndex, desc) + return + } + ts.expFail[server]-- +} + +func (ts *TestScheduler) Request(testIndex, reqIndex int) requestWithID { + if len(ts.sent[testIndex]) < reqIndex { + ts.t.Errorf("Missing request from test case %d index %d", testIndex, reqIndex) + return requestWithID{} + } + return ts.sent[testIndex][reqIndex-1] +} + +func (ts *TestScheduler) ServerEvent(evType *request.EventType, server request.Server, data any) { + ts.events = append(ts.events, request.Event{ + Type: evType, + Server: server, + Data: data, + }) +} + +func (ts *TestScheduler) RequestEvent(evType *request.EventType, req requestWithID, resp request.Response) { + if req.request == nil { + return + } + ts.events = append(ts.events, request.Event{ + Type: evType, + Server: req.sid.Server, + Data: request.RequestResponse{ + ID: req.sid.ID, + Request: req.request, + Response: resp, + }, + }) +} + +func (ts *TestScheduler) AddServer(server request.Server, allowance int) { + ts.servers = append(ts.servers, server) + ts.allowance[server] = allowance + ts.ServerEvent(request.EvRegistered, server, nil) +} + +func (ts *TestScheduler) RemoveServer(server request.Server) { + ts.servers = append(ts.servers, server) + for i, s := range ts.servers { + if s == server { + copy(ts.servers[i:len(ts.servers)-1], ts.servers[i+1:]) + ts.servers = ts.servers[:len(ts.servers)-1] + break + } + } + delete(ts.allowance, server) + ts.ServerEvent(request.EvUnregistered, server, nil) +} + +func (ts *TestScheduler) AddAllowance(server request.Server, allowance int) { + ts.allowance[server] += allowance +} + +func (ts *TestScheduler) ExpFail(server request.Server) { + ts.expFail[server]++ +} + +type TestCommitteeChain struct { + fsp, nsp uint64 + init bool +} + +func (t *TestCommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error { + t.fsp, t.nsp, t.init = bootstrap.Header.SyncPeriod(), bootstrap.Header.SyncPeriod()+2, true + return nil +} + +func (t *TestCommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error { + period := update.AttestedHeader.Header.SyncPeriod() + if period < t.fsp || period > t.nsp || !t.init { + return light.ErrInvalidPeriod + } + if period == t.nsp { + t.nsp++ + } + return nil +} + +func (t *TestCommitteeChain) NextSyncPeriod() (uint64, bool) { + return t.nsp, t.init +} + +func (tc *TestCommitteeChain) ExpInit(t *testing.T, ExpInit bool) { + if tc.init != ExpInit { + t.Errorf("Incorrect init flag (expected %v, got %v)", ExpInit, tc.init) + } +} + +func (t *TestCommitteeChain) SetNextSyncPeriod(nsp uint64) { + t.init, t.nsp = true, nsp +} + +func (tc *TestCommitteeChain) ExpNextSyncPeriod(t *testing.T, expNsp uint64) { + tc.ExpInit(t, true) + if tc.nsp != expNsp { + t.Errorf("Incorrect NextSyncPeriod (expected %d, got %d)", expNsp, tc.nsp) + } +} + +type TestHeadTracker struct { + phead types.HeadInfo + validated []types.SignedHeader +} + +func (ht *TestHeadTracker) ValidateHead(head types.SignedHeader) (bool, error) { + ht.validated = append(ht.validated, head) + return true, nil +} + +// TODO add test case for finality +func (ht *TestHeadTracker) ValidateFinality(head types.FinalityUpdate) (bool, error) { + return true, nil +} + +func (ht *TestHeadTracker) ExpValidated(t *testing.T, tci int, expHeads []types.SignedHeader) { + for i, expHead := range expHeads { + if i >= len(ht.validated) { + t.Errorf("Missing validated head in test case #%d index #%d (expected {slot %d blockRoot %x}, got none)", tci, i, expHead.Header.Slot, expHead.Header.Hash()) + continue + } + if ht.validated[i] != expHead { + vhead := ht.validated[i].Header + t.Errorf("Wrong validated head in test case #%d index #%d (expected {slot %d blockRoot %x}, got {slot %d blockRoot %x})", tci, i, expHead.Header.Slot, expHead.Header.Hash(), vhead.Slot, vhead.Hash()) + } + } + for i := len(expHeads); i < len(ht.validated); i++ { + vhead := ht.validated[i].Header + t.Errorf("Unexpected validated head in test case #%d index #%d (expected none, got {slot %d blockRoot %x})", tci, i, vhead.Slot, vhead.Hash()) + } + ht.validated = nil +} + +func (ht *TestHeadTracker) SetPrefetchHead(head types.HeadInfo) { + ht.phead = head +} + +func (ht *TestHeadTracker) ExpPrefetch(t *testing.T, tci int, exp types.HeadInfo) { + if ht.phead != exp { + t.Errorf("Wrong prefetch head in test case #%d (expected {slot %d blockRoot %x}, got {slot %d blockRoot %x})", tci, exp.Slot, exp.BlockRoot, ht.phead.Slot, ht.phead.BlockRoot) + } +} diff --git a/beacon/light/sync/types.go b/beacon/light/sync/types.go new file mode 100644 index 000000000000..6449ae842d00 --- /dev/null +++ b/beacon/light/sync/types.go @@ -0,0 +1,42 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" +) + +var ( + EvNewHead = &request.EventType{Name: "newHead"} // data: types.HeadInfo + EvNewSignedHead = &request.EventType{Name: "newSignedHead"} // data: types.SignedHeader + EvNewFinalityUpdate = &request.EventType{Name: "newFinalityUpdate"} // data: types.FinalityUpdate +) + +type ( + ReqUpdates struct { + FirstPeriod, Count uint64 + } + RespUpdates struct { + Updates []*types.LightClientUpdate + Committees []*types.SerializedSyncCommittee + } + ReqHeader common.Hash + ReqCheckpointData common.Hash + ReqBeaconBlock common.Hash +) diff --git a/beacon/light/sync/update_sync.go b/beacon/light/sync/update_sync.go new file mode 100644 index 000000000000..533e470fb022 --- /dev/null +++ b/beacon/light/sync/update_sync.go @@ -0,0 +1,299 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "sort" + + "github.com/ethereum/go-ethereum/beacon/light" + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +const maxUpdateRequest = 8 // maximum number of updates requested in a single request + +type committeeChain interface { + CheckpointInit(bootstrap types.BootstrapData) error + InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error + NextSyncPeriod() (uint64, bool) +} + +// CheckpointInit implements request.Module; it fetches the light client bootstrap +// data belonging to the given checkpoint hash and initializes the committee chain +// if successful. +type CheckpointInit struct { + chain committeeChain + checkpointHash common.Hash + locked request.ServerAndID + initialized bool +} + +// NewCheckpointInit creates a new CheckpointInit. +func NewCheckpointInit(chain committeeChain, checkpointHash common.Hash) *CheckpointInit { + return &CheckpointInit{ + chain: chain, + checkpointHash: checkpointHash, + } +} + +// Process implements request.Module. +func (s *CheckpointInit) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + if !event.IsRequestEvent() { + continue + } + sid, req, resp := event.RequestInfo() + if s.locked == sid { + s.locked = request.ServerAndID{} + } + if resp != nil { + if checkpoint := resp.(*types.BootstrapData); checkpoint.Header.Hash() == common.Hash(req.(ReqCheckpointData)) { + s.chain.CheckpointInit(*checkpoint) + s.initialized = true + return + } + + requester.Fail(event.Server, "invalid checkpoint data") + } + } + // start a request if possible + if s.initialized || s.locked != (request.ServerAndID{}) { + return + } + cs := requester.CanSendTo() + if len(cs) == 0 { + return + } + server := cs[0] + id := requester.Send(server, ReqCheckpointData(s.checkpointHash)) + s.locked = request.ServerAndID{Server: server, ID: id} +} + +// ForwardUpdateSync implements request.Module; it fetches updates between the +// committee chain head and each server's announced head. Updates are fetched +// in batches and multiple batches can also be requested in parallel. +// Out of order responses are also handled; if a batch of updates cannot be added +// to the chain immediately because of a gap then the future updates are +// remembered until they can be processed. +type ForwardUpdateSync struct { + chain committeeChain + rangeLock rangeLock + lockedIDs map[request.ServerAndID]struct{} + processQueue []updateResponse + nextSyncPeriod map[request.Server]uint64 +} + +// NewForwardUpdateSync creates a new ForwardUpdateSync. +func NewForwardUpdateSync(chain committeeChain) *ForwardUpdateSync { + return &ForwardUpdateSync{ + chain: chain, + rangeLock: make(rangeLock), + lockedIDs: make(map[request.ServerAndID]struct{}), + nextSyncPeriod: make(map[request.Server]uint64), + } +} + +// rangeLock allows locking sections of an integer space, preventing the syncing +// mechanism from making requests again for sections where a not timed out request +// is already pending or where already fetched and unprocessed data is available. +type rangeLock map[uint64]int + +// lock locks or unlocks the given section, depending on the sign of the add parameter. +func (r rangeLock) lock(first, count uint64, add int) { + for i := first; i < first+count; i++ { + if v := r[i] + add; v > 0 { + r[i] = v + } else { + delete(r, i) + } + } +} + +// firstUnlocked returns the first unlocked section starting at or after start +// and not longer than maxCount. +func (r rangeLock) firstUnlocked(start, maxCount uint64) (first, count uint64) { + first = start + for { + if _, ok := r[first]; !ok { + break + } + first++ + } + for { + count++ + if count == maxCount { + break + } + if _, ok := r[first+count]; ok { + break + } + } + return +} + +// lockRange locks the range belonging to the given update request, unless the +// same request has already been locked +func (s *ForwardUpdateSync) lockRange(sid request.ServerAndID, req ReqUpdates) { + if _, ok := s.lockedIDs[sid]; ok { + return + } + s.lockedIDs[sid] = struct{}{} + s.rangeLock.lock(req.FirstPeriod, req.Count, 1) +} + +// unlockRange unlocks the range belonging to the given update request, unless +// same request has already been unlocked +func (s *ForwardUpdateSync) unlockRange(sid request.ServerAndID, req ReqUpdates) { + if _, ok := s.lockedIDs[sid]; !ok { + return + } + delete(s.lockedIDs, sid) + s.rangeLock.lock(req.FirstPeriod, req.Count, -1) +} + +// verifyRange returns true if the number of updates and the individual update +// periods in the response match the requested section. +func (s *ForwardUpdateSync) verifyRange(request ReqUpdates, response RespUpdates) bool { + if uint64(len(response.Updates)) != request.Count || uint64(len(response.Committees)) != request.Count { + return false + } + for i, update := range response.Updates { + if update.AttestedHeader.Header.SyncPeriod() != request.FirstPeriod+uint64(i) { + return false + } + } + return true +} + +// updateResponse is a response that has passed initial verification and has been +// queued for processing. Note that an update response cannot be processed until +// the previous updates have also been added to the chain. +type updateResponse struct { + sid request.ServerAndID + request ReqUpdates + response RespUpdates +} + +// updateResponseList implements sort.Sort and sorts update request/response events by FirstPeriod. +type updateResponseList []updateResponse + +func (u updateResponseList) Len() int { return len(u) } +func (u updateResponseList) Swap(i, j int) { u[i], u[j] = u[j], u[i] } +func (u updateResponseList) Less(i, j int) bool { + return u[i].request.FirstPeriod < u[j].request.FirstPeriod +} + +// Process implements request.Module. +func (s *ForwardUpdateSync) Process(requester request.Requester, events []request.Event) { + for _, event := range events { + switch event.Type { + case request.EvResponse, request.EvFail, request.EvTimeout: + sid, rq, rs := event.RequestInfo() + req := rq.(ReqUpdates) + var queued bool + if event.Type == request.EvResponse { + resp := rs.(RespUpdates) + if s.verifyRange(req, resp) { + // there is a response with a valid format; put it in the process queue + s.processQueue = append(s.processQueue, updateResponse{sid: sid, request: req, response: resp}) + s.lockRange(sid, req) + queued = true + } else { + requester.Fail(event.Server, "invalid update range") + } + } + if !queued { + s.unlockRange(sid, req) + } + case EvNewSignedHead: + signedHead := event.Data.(types.SignedHeader) + s.nextSyncPeriod[event.Server] = types.SyncPeriod(signedHead.SignatureSlot + 256) + case request.EvUnregistered: + delete(s.nextSyncPeriod, event.Server) + } + } + + // try processing ordered list of available responses + sort.Sort(updateResponseList(s.processQueue)) + for s.processQueue != nil { + u := s.processQueue[0] + if !s.processResponse(requester, u) { + break + } + s.unlockRange(u.sid, u.request) + s.processQueue = s.processQueue[1:] + if len(s.processQueue) == 0 { + s.processQueue = nil + } + } + + // start new requests if possible + startPeriod, chainInit := s.chain.NextSyncPeriod() + if !chainInit { + return + } + for { + firstPeriod, maxCount := s.rangeLock.firstUnlocked(startPeriod, maxUpdateRequest) + var ( + sendTo request.Server + bestCount uint64 + ) + for _, server := range requester.CanSendTo() { + nextPeriod := s.nextSyncPeriod[server] + if nextPeriod <= firstPeriod { + continue + } + count := maxCount + if nextPeriod < firstPeriod+maxCount { + count = nextPeriod - firstPeriod + } + if count > bestCount { + sendTo, bestCount = server, count + } + } + if sendTo == nil { + return + } + req := ReqUpdates{FirstPeriod: firstPeriod, Count: bestCount} + id := requester.Send(sendTo, req) + s.lockRange(request.ServerAndID{Server: sendTo, ID: id}, req) + } +} + +// processResponse adds the fetched updates and committees to the committee chain. +// Returns true in case of full or partial success. +func (s *ForwardUpdateSync) processResponse(requester request.Requester, u updateResponse) (success bool) { + for i, update := range u.response.Updates { + if err := s.chain.InsertUpdate(update, u.response.Committees[i]); err != nil { + if err == light.ErrInvalidPeriod { + // there is a gap in the update periods; stop processing without + // failing and try again next time + return + } + if err == light.ErrInvalidUpdate || err == light.ErrWrongCommitteeRoot || err == light.ErrCannotReorg { + requester.Fail(u.sid.Server, "invalid update received") + } else { + log.Error("Unexpected InsertUpdate error", "error", err) + } + return + } + success = true + } + return +} diff --git a/beacon/light/sync/update_sync_test.go b/beacon/light/sync/update_sync_test.go new file mode 100644 index 000000000000..1c4b3d6d76fa --- /dev/null +++ b/beacon/light/sync/update_sync_test.go @@ -0,0 +1,219 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package sync + +import ( + "testing" + + "github.com/ethereum/go-ethereum/beacon/light/request" + "github.com/ethereum/go-ethereum/beacon/types" +) + +func TestCheckpointInit(t *testing.T) { + chain := &TestCommitteeChain{} + checkpoint := &types.BootstrapData{Header: types.Header{Slot: 0x2000*4 + 0x1000}} // period 4 + checkpointHash := checkpoint.Header.Hash() + chkInit := NewCheckpointInit(chain, checkpointHash) + ts := NewTestScheduler(t, chkInit) + // add 2 servers + ts.AddServer(testServer1, 1) + ts.AddServer(testServer2, 1) + + // expect bootstrap request to server 1 + ts.Run(1, testServer1, ReqCheckpointData(checkpointHash)) + + // server 1 times out; expect request to server 2 + ts.RequestEvent(request.EvTimeout, ts.Request(1, 1), nil) + ts.Run(2, testServer2, ReqCheckpointData(checkpointHash)) + + // invalid response from server 2; expect init state to still be false + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), &types.BootstrapData{Header: types.Header{Slot: 123456}}) + ts.ExpFail(testServer2) + ts.Run(3) + chain.ExpInit(t, false) + + // server 1 fails (hard timeout) + ts.RequestEvent(request.EvFail, ts.Request(1, 1), nil) + ts.Run(4) + chain.ExpInit(t, false) + + // server 3 is registered; expect bootstrap request to server 3 + ts.AddServer(testServer3, 1) + ts.Run(5, testServer3, ReqCheckpointData(checkpointHash)) + + // valid response from server 3; expect chain to be initialized + ts.RequestEvent(request.EvResponse, ts.Request(5, 1), checkpoint) + ts.Run(6) + chain.ExpInit(t, true) +} + +func TestUpdateSyncParallel(t *testing.T) { + chain := &TestCommitteeChain{} + chain.SetNextSyncPeriod(0) + updateSync := NewForwardUpdateSync(chain) + ts := NewTestScheduler(t, updateSync) + // add 2 servers, head at period 100; allow 3-3 parallel requests for each + ts.AddServer(testServer1, 3) + ts.ServerEvent(EvNewSignedHead, testServer1, types.SignedHeader{SignatureSlot: 0x2000*100 + 0x1000}) + ts.AddServer(testServer2, 3) + ts.ServerEvent(EvNewSignedHead, testServer2, types.SignedHeader{SignatureSlot: 0x2000*100 + 0x1000}) + + // expect 6 requests to be sent + ts.Run(1, + testServer1, ReqUpdates{FirstPeriod: 0, Count: 8}, + testServer1, ReqUpdates{FirstPeriod: 8, Count: 8}, + testServer1, ReqUpdates{FirstPeriod: 16, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 24, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 32, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 40, Count: 8}) + + // valid response to request 1; expect 8 periods synced and a new request started + ts.RequestEvent(request.EvResponse, ts.Request(1, 1), testRespUpdate(ts.Request(1, 1))) + ts.AddAllowance(testServer1, 1) + ts.Run(2, testServer1, ReqUpdates{FirstPeriod: 48, Count: 8}) + chain.ExpNextSyncPeriod(t, 8) + + // valid response to requests 4 and 5 + ts.RequestEvent(request.EvResponse, ts.Request(1, 4), testRespUpdate(ts.Request(1, 4))) + ts.RequestEvent(request.EvResponse, ts.Request(1, 5), testRespUpdate(ts.Request(1, 5))) + ts.AddAllowance(testServer2, 2) + // expect 2 more requests but no sync progress (responses 4 and 5 cannot be added before 2 and 3) + ts.Run(3, + testServer2, ReqUpdates{FirstPeriod: 56, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 64, Count: 8}) + chain.ExpNextSyncPeriod(t, 8) + + // soft timeout for requests 2 and 3 (server 1 is overloaded) + ts.RequestEvent(request.EvTimeout, ts.Request(1, 2), nil) + ts.RequestEvent(request.EvTimeout, ts.Request(1, 3), nil) + // no allowance, no more requests + ts.Run(4) + + // valid response to requests 6 and 8 and 9 + ts.RequestEvent(request.EvResponse, ts.Request(1, 6), testRespUpdate(ts.Request(1, 6))) + ts.RequestEvent(request.EvResponse, ts.Request(3, 1), testRespUpdate(ts.Request(3, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(3, 2), testRespUpdate(ts.Request(3, 2))) + ts.AddAllowance(testServer2, 3) + // server 2 can now resend requests 2 and 3 (timed out by server 1) and also send a new one + ts.Run(5, + testServer2, ReqUpdates{FirstPeriod: 8, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 16, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 72, Count: 8}) + + // server 1 finally answers timed out request 2 + ts.RequestEvent(request.EvResponse, ts.Request(1, 2), testRespUpdate(ts.Request(1, 2))) + ts.AddAllowance(testServer1, 1) + // expect sync progress and one new request + ts.Run(6, testServer1, ReqUpdates{FirstPeriod: 80, Count: 8}) + chain.ExpNextSyncPeriod(t, 16) + + // server 2 answers requests 11 and 12 (resends of requests 2 and 3) + ts.RequestEvent(request.EvResponse, ts.Request(5, 1), testRespUpdate(ts.Request(5, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(5, 2), testRespUpdate(ts.Request(5, 2))) + ts.AddAllowance(testServer2, 2) + ts.Run(7, + testServer2, ReqUpdates{FirstPeriod: 88, Count: 8}, + testServer2, ReqUpdates{FirstPeriod: 96, Count: 4}) + // finally the gap is filled, update can process responses up to req6 + chain.ExpNextSyncPeriod(t, 48) + + // all remaining requests are answered + ts.RequestEvent(request.EvResponse, ts.Request(1, 3), testRespUpdate(ts.Request(1, 3))) + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testRespUpdate(ts.Request(2, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(5, 3), testRespUpdate(ts.Request(5, 3))) + ts.RequestEvent(request.EvResponse, ts.Request(6, 1), testRespUpdate(ts.Request(6, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testRespUpdate(ts.Request(7, 1))) + ts.RequestEvent(request.EvResponse, ts.Request(7, 2), testRespUpdate(ts.Request(7, 2))) + ts.Run(8) + // expect chain to be fully synced + chain.ExpNextSyncPeriod(t, 100) +} + +func TestUpdateSyncDifferentHeads(t *testing.T) { + chain := &TestCommitteeChain{} + chain.SetNextSyncPeriod(10) + updateSync := NewForwardUpdateSync(chain) + ts := NewTestScheduler(t, updateSync) + // add 3 servers with different announced head periods + ts.AddServer(testServer1, 1) + ts.ServerEvent(EvNewSignedHead, testServer1, types.SignedHeader{SignatureSlot: 0x2000*15 + 0x1000}) + ts.AddServer(testServer2, 1) + ts.ServerEvent(EvNewSignedHead, testServer2, types.SignedHeader{SignatureSlot: 0x2000*16 + 0x1000}) + ts.AddServer(testServer3, 1) + ts.ServerEvent(EvNewSignedHead, testServer3, types.SignedHeader{SignatureSlot: 0x2000*17 + 0x1000}) + + // expect request to the best announced head + ts.Run(1, testServer3, ReqUpdates{FirstPeriod: 10, Count: 7}) + + // request times out, expect request to the next best head + ts.RequestEvent(request.EvTimeout, ts.Request(1, 1), nil) + ts.Run(2, testServer2, ReqUpdates{FirstPeriod: 10, Count: 6}) + + // request times out, expect request to the last available server + ts.RequestEvent(request.EvTimeout, ts.Request(2, 1), nil) + ts.Run(3, testServer1, ReqUpdates{FirstPeriod: 10, Count: 5}) + + // valid response to request 3, expect chain synced to period 15 + ts.RequestEvent(request.EvResponse, ts.Request(3, 1), testRespUpdate(ts.Request(3, 1))) + ts.AddAllowance(testServer1, 1) + ts.Run(4) + chain.ExpNextSyncPeriod(t, 15) + + // invalid response to request 1, server can only deliver updates up to period 15 despite announced head + truncated := ts.Request(1, 1) + truncated.request = ReqUpdates{FirstPeriod: 10, Count: 5} + ts.RequestEvent(request.EvResponse, ts.Request(1, 1), testRespUpdate(truncated)) + ts.ExpFail(testServer3) + ts.Run(5) + // expect no progress of chain head + chain.ExpNextSyncPeriod(t, 15) + + // valid response to request 2, expect chain synced to period 16 + ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testRespUpdate(ts.Request(2, 1))) + ts.AddAllowance(testServer2, 1) + ts.Run(6) + chain.ExpNextSyncPeriod(t, 16) + + // a new server is registered with announced head period 17 + ts.AddServer(testServer4, 1) + ts.ServerEvent(EvNewSignedHead, testServer4, types.SignedHeader{SignatureSlot: 0x2000*17 + 0x1000}) + // expect request to sync one more period + ts.Run(7, testServer4, ReqUpdates{FirstPeriod: 16, Count: 1}) + + // valid response, expect chain synced to period 17 + ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testRespUpdate(ts.Request(7, 1))) + ts.AddAllowance(testServer4, 1) + ts.Run(8) + chain.ExpNextSyncPeriod(t, 17) +} + +func testRespUpdate(request requestWithID) request.Response { + var resp RespUpdates + if request.request == nil { + return resp + } + req := request.request.(ReqUpdates) + resp.Updates = make([]*types.LightClientUpdate, int(req.Count)) + resp.Committees = make([]*types.SerializedSyncCommittee, int(req.Count)) + period := req.FirstPeriod + for i := range resp.Updates { + resp.Updates[i] = &types.LightClientUpdate{AttestedHeader: types.SignedHeader{Header: types.Header{Slot: 0x2000*period + 0x1000}}} + resp.Committees[i] = new(types.SerializedSyncCommittee) + period++ + } + return resp +} diff --git a/beacon/params/params.go b/beacon/params/params.go index ee9feb1acbea..e4e0d009340e 100644 --- a/beacon/params/params.go +++ b/beacon/params/params.go @@ -41,4 +41,6 @@ const ( StateIndexNextSyncCommittee = 55 StateIndexExecPayload = 56 StateIndexExecHead = 908 + + BodyIndexExecPayload = 25 ) diff --git a/beacon/types/light_sync.go b/beacon/types/light_sync.go index 3284081e4d49..ed62d237f126 100644 --- a/beacon/types/light_sync.go +++ b/beacon/types/light_sync.go @@ -20,11 +20,20 @@ import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/beacon/merkle" "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/common" + "github.com/protolambda/zrnt/eth2/beacon/capella" + "github.com/protolambda/ztyp/tree" ) +// HeadInfo represents an unvalidated new head announcement. +type HeadInfo struct { + Slot uint64 + BlockRoot common.Hash +} + // BootstrapData contains a sync committee where light sync can be started, // together with a proof through a beacon header and corresponding state. // Note: BootstrapData is fetched from a server based on a known checkpoint hash. @@ -134,3 +143,50 @@ func (u UpdateScore) BetterThan(w UpdateScore) bool { } return u.SignerCount > w.SignerCount } + +type HeaderWithExecProof struct { + Header + PayloadHeader *capella.ExecutionPayloadHeader + PayloadBranch merkle.Values +} + +func (h *HeaderWithExecProof) Validate() error { + payloadRoot := merkle.Value(h.PayloadHeader.HashTreeRoot(tree.GetHashFn())) + return merkle.VerifyProof(h.BodyRoot, params.BodyIndexExecPayload, h.PayloadBranch, payloadRoot) +} + +type FinalityUpdate struct { + Attested, Finalized HeaderWithExecProof + FinalityBranch merkle.Values + // Sync committee BLS signature aggregate + Signature SyncAggregate + // Slot in which the signature has been created (newer than Header.Slot, + // determines the signing sync committee) + SignatureSlot uint64 +} + +func (u *FinalityUpdate) SignedHeader() SignedHeader { + return SignedHeader{ + Header: u.Attested.Header, + Signature: u.Signature, + SignatureSlot: u.SignatureSlot, + } +} + +func (u *FinalityUpdate) Validate() error { + if err := u.Attested.Validate(); err != nil { + return err + } + if err := u.Finalized.Validate(); err != nil { + return err + } + return merkle.VerifyProof(u.Attested.StateRoot, params.StateIndexFinalBlock, u.FinalityBranch, merkle.Value(u.Finalized.Hash())) +} + +// ChainHeadEvent returns an authenticated execution payload associated with the +// latest accepted head of the beacon chain, along with the hash of the latest +// finalized execution block. +type ChainHeadEvent struct { + HeadBlock *engine.ExecutableData + Finalized common.Hash +} diff --git a/cmd/blsync/engine_api.go b/cmd/blsync/engine_api.go new file mode 100644 index 000000000000..d10750e295fe --- /dev/null +++ b/cmd/blsync/engine_api.go @@ -0,0 +1,69 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" +) + +func updateEngineApi(client *rpc.Client, headCh chan types.ChainHeadEvent) { + for event := range headCh { + if client == nil { // dry run, no engine API specified + log.Info("New execution block retrieved", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "finalized block hash", event.Finalized) + } else { + if status, err := callNewPayloadV2(client, event.HeadBlock); err == nil { + log.Info("Successful NewPayload", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "status", status) + } else { + log.Error("Failed NewPayload", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "error", err) + } + if status, err := callForkchoiceUpdatedV1(client, event.HeadBlock.BlockHash, event.Finalized); err == nil { + log.Info("Successful ForkchoiceUpdated", "head", event.HeadBlock.BlockHash, "status", status) + } else { + log.Error("Failed ForkchoiceUpdated", "head", event.HeadBlock.BlockHash, "error", err) + } + } + } +} + +func callNewPayloadV2(client *rpc.Client, execData *engine.ExecutableData) (string, error) { + var resp engine.PayloadStatusV1 + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + err := client.CallContext(ctx, &resp, "engine_newPayloadV2", execData) + cancel() + return resp.Status, err +} + +func callForkchoiceUpdatedV1(client *rpc.Client, headHash, finalizedHash common.Hash) (string, error) { + var resp engine.ForkChoiceResponse + update := engine.ForkchoiceStateV1{ + HeadBlockHash: headHash, + SafeBlockHash: finalizedHash, + FinalizedBlockHash: finalizedHash, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + err := client.CallContext(ctx, &resp, "engine_forkchoiceUpdatedV1", update, nil) + cancel() + return resp.PayloadStatus.Status, err +} diff --git a/cmd/blsync/main.go b/cmd/blsync/main.go new file mode 100644 index 000000000000..fd22761d3c45 --- /dev/null +++ b/cmd/blsync/main.go @@ -0,0 +1,125 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/ethereum/go-ethereum/beacon/blsync" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rpc" + "github.com/mattn/go-colorable" + "github.com/mattn/go-isatty" + "github.com/urfave/cli/v2" +) + +var ( + verbosityFlag = &cli.IntFlag{ + Name: "verbosity", + Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail", + Value: 3, + Category: flags.LoggingCategory, + } + vmoduleFlag = &cli.StringFlag{ + Name: "vmodule", + Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", + Value: "", + Hidden: true, + Category: flags.LoggingCategory, + } +) + +func main() { + app := flags.NewApp("beacon light syncer tool") + app.Flags = []cli.Flag{ + utils.BeaconApiFlag, + utils.BeaconApiHeaderFlag, + utils.BeaconThresholdFlag, + utils.BeaconNoFilterFlag, + utils.BeaconConfigFlag, + utils.BeaconGenesisRootFlag, + utils.BeaconGenesisTimeFlag, + utils.BeaconCheckpointFlag, + //TODO datadir for optional permanent database + utils.MainnetFlag, + utils.SepoliaFlag, + utils.GoerliFlag, + utils.BlsyncApiFlag, + utils.BlsyncJWTSecretFlag, + verbosityFlag, + vmoduleFlag, + } + app.Action = sync + + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func sync(ctx *cli.Context) error { + usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" + output := io.Writer(os.Stderr) + if usecolor { + output = colorable.NewColorable(os.Stderr) + } + verbosity := log.FromLegacyLevel(ctx.Int(verbosityFlag.Name)) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(output, verbosity, usecolor))) + + headCh := make(chan types.ChainHeadEvent, 16) + client := blsync.NewClient(ctx) + sub := client.SubscribeChainHeadEvent(headCh) + go updateEngineApi(makeRPCClient(ctx), headCh) + client.Start() + // run until stopped + <-ctx.Done() + client.Stop() + sub.Unsubscribe() + close(headCh) + return nil +} + +func makeRPCClient(ctx *cli.Context) *rpc.Client { + if !ctx.IsSet(utils.BlsyncApiFlag.Name) { + log.Warn("No engine API target specified, performing a dry run") + return nil + } + if !ctx.IsSet(utils.BlsyncJWTSecretFlag.Name) { + utils.Fatalf("JWT secret parameter missing") //TODO use default if datadir is specified + } + + engineApiUrl, jwtFileName := ctx.String(utils.BlsyncApiFlag.Name), ctx.String(utils.BlsyncJWTSecretFlag.Name) + var jwtSecret [32]byte + if jwt, err := node.ObtainJWTSecret(jwtFileName); err == nil { + copy(jwtSecret[:], jwt) + } else { + utils.Fatalf("Error loading or generating JWT secret: %v", err) + } + auth := node.NewJWTAuth(jwtSecret) + cl, err := rpc.DialOptions(context.Background(), engineApiUrl, rpc.WithHTTPAuth(auth)) + if err != nil { + utils.Fatalf("Could not create RPC client: %v", err) + } + return cl +} diff --git a/cmd/clef/README.md b/cmd/clef/README.md index 3a43db8c95a3..cf0926513603 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -916,7 +916,7 @@ There are a couple of implementation for a UI. We'll try to keep this list up to | Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters| | ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- | -| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| -| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | -| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | -| Clef UI| https://github.com/ethereum/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| +| QtSigner| https://github.com/holiman/qtsigner/ | Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| +| GtkSigner| https://github.com/holiman/gtksigner | Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | +| Frame | https://github.com/floating/frame/commits/go-signer | Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | +| Clef UI| https://github.com/ethereum/clef-ui | Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index e8b3725b17a5..a34a41dacdaa 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -26,7 +26,7 @@ import ( "io" "math/big" "os" - "path" + "path/filepath" "sort" "strings" @@ -56,21 +56,21 @@ type Chain struct { // NewChain takes the given chain.rlp file, and decodes and returns // the blocks from the file. func NewChain(dir string) (*Chain, error) { - gen, err := loadGenesis(path.Join(dir, "genesis.json")) + gen, err := loadGenesis(filepath.Join(dir, "genesis.json")) if err != nil { return nil, err } gblock := gen.ToBlock() - blocks, err := blocksFromFile(path.Join(dir, "chain.rlp"), gblock) + blocks, err := blocksFromFile(filepath.Join(dir, "chain.rlp"), gblock) if err != nil { return nil, err } - state, err := readState(path.Join(dir, "headstate.json")) + state, err := readState(filepath.Join(dir, "headstate.json")) if err != nil { return nil, err } - accounts, err := readAccounts(path.Join(dir, "accounts.json")) + accounts, err := readAccounts(filepath.Join(dir, "accounts.json")) if err != nil { return nil, err } diff --git a/cmd/devp2p/internal/ethtest/engine.go b/cmd/devp2p/internal/ethtest/engine.go index ea4fc76e6ff7..0e94efa5bdac 100644 --- a/cmd/devp2p/internal/ethtest/engine.go +++ b/cmd/devp2p/internal/ethtest/engine.go @@ -22,7 +22,7 @@ import ( "io" "net/http" "os" - "path" + "path/filepath" "time" "github.com/ethereum/go-ethereum/common" @@ -38,7 +38,7 @@ type EngineClient struct { // NewEngineClient creates a new engine client. func NewEngineClient(dir, url, jwt string) (*EngineClient, error) { - headfcu, err := os.ReadFile(path.Join(dir, "headfcu.json")) + headfcu, err := os.ReadFile(filepath.Join(dir, "headfcu.json")) if err != nil { return nil, fmt.Errorf("failed to read headfcu: %w", err) } diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index 64e063358545..8ff3f1f71a6e 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -648,7 +648,7 @@ The server should reject the request.`, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8}}, }, nBytes: 5000, - expHashes: []common.Hash{common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")}, + expHashes: []common.Hash{types.EmptyCodeHash}, }, { diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index d9efe2624432..b5cc27a2b590 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -754,8 +754,8 @@ func makeSidecar(data ...byte) *types.BlobTxSidecar { ) for i := range blobs { blobs[i][0] = data[i] - c, _ := kzg4844.BlobToCommitment(blobs[i]) - p, _ := kzg4844.ComputeBlobProof(blobs[i], c) + c, _ := kzg4844.BlobToCommitment(&blobs[i]) + p, _ := kzg4844.ComputeBlobProof(&blobs[i], c) commitments = append(commitments, c) proofs = append(proofs, p) } diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index ad73bc9f90e7..d70adda51f92 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -20,7 +20,7 @@ import ( crand "crypto/rand" "fmt" "os" - "path" + "path/filepath" "testing" "time" @@ -39,7 +39,7 @@ func makeJWTSecret() (string, [32]byte, error) { if _, err := crand.Read(secret[:]); err != nil { return "", secret, fmt.Errorf("failed to create jwt secret: %v", err) } - jwtPath := path.Join(os.TempDir(), "jwt_secret") + jwtPath := filepath.Join(os.TempDir(), "jwt_secret") if err := os.WriteFile(jwtPath, []byte(hexutil.Encode(secret[:])), 0600); err != nil { return "", secret, fmt.Errorf("failed to prepare jwt secret file: %v", err) } diff --git a/cmd/devp2p/main.go b/cmd/devp2p/main.go index 8461a8b9b5e2..66974bba5890 100644 --- a/cmd/devp2p/main.go +++ b/cmd/devp2p/main.go @@ -66,9 +66,15 @@ func commandHasFlag(ctx *cli.Context, flag cli.Flag) bool { for _, name := range names { set[name] = struct{}{} } - for _, fn := range ctx.FlagNames() { - if _, ok := set[fn]; ok { - return true + for _, ctx := range ctx.Lineage() { + if ctx.Command != nil { + for _, f := range ctx.Command.Flags { + for _, name := range f.Names() { + if _, ok := set[name]; ok { + return true + } + } + } } } return false diff --git a/cmd/era/main.go b/cmd/era/main.go index e27d8ccec605..8b57fd695c53 100644 --- a/cmd/era/main.go +++ b/cmd/era/main.go @@ -18,10 +18,11 @@ package main import ( "encoding/json" + "errors" "fmt" "math/big" "os" - "path" + "path/filepath" "strconv" "strings" "time" @@ -175,14 +176,14 @@ func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { if epoch >= uint64(len(entries)) { return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch) } - return era.Open(path.Join(dir, entries[epoch])) + return era.Open(filepath.Join(dir, entries[epoch])) } // verify checks each era1 file in a directory to ensure it is well-formed and // that the accumulator matches the expected value. func verify(ctx *cli.Context) error { if ctx.Args().Len() != 1 { - return fmt.Errorf("missing accumulators file") + return errors.New("missing accumulators file") } roots, err := readHashes(ctx.Args().First()) @@ -203,7 +204,7 @@ func verify(ctx *cli.Context) error { } if len(entries) != len(roots) { - return fmt.Errorf("number of era1 files should match the number of accumulator hashes") + return errors.New("number of era1 files should match the number of accumulator hashes") } // Verify each epoch matches the expected root. @@ -211,7 +212,7 @@ func verify(ctx *cli.Context) error { // Wrap in function so defers don't stack. err := func() error { name := entries[i] - e, err := era.Open(path.Join(dir, name)) + e, err := era.Open(filepath.Join(dir, name)) if err != nil { return fmt.Errorf("error opening era1 file %s: %w", name, err) } @@ -308,7 +309,7 @@ func checkAccumulator(e *era.Era) error { func readHashes(f string) ([]common.Hash, error) { b, err := os.ReadFile(f) if err != nil { - return nil, fmt.Errorf("unable to open accumulators file") + return nil, errors.New("unable to open accumulators file") } s := strings.Split(string(b), "\n") // Remove empty last element, if present. diff --git a/cmd/evm/internal/t8ntool/block.go b/cmd/evm/internal/t8ntool/block.go index a2dc4734372b..62c8593a1d47 100644 --- a/cmd/evm/internal/t8ntool/block.go +++ b/cmd/evm/internal/t8ntool/block.go @@ -242,7 +242,7 @@ func readInput(ctx *cli.Context) (*bbInput, error) { if headerStr == stdinSelector || ommersStr == stdinSelector || txsStr == stdinSelector || cliqueStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { - return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) } } if cliqueStr != stdinSelector && cliqueStr != "" { diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 8533b7863769..7f66ba4d85d6 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -86,7 +86,7 @@ func Transaction(ctx *cli.Context) error { if txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) } // Decode the body of already signed transactions body = common.FromHex(inputData.TxRlp) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 7802d4965199..a9489d069a70 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -22,7 +22,7 @@ import ( "fmt" "math/big" "os" - "path" + "path/filepath" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -96,7 +96,7 @@ func Transition(ctx *cli.Context) error { Debug: true, } getTracer = func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) { - traceFile, err := os.Create(path.Join(baseDir, fmt.Sprintf("trace-%d-%v.jsonl", txIndex, txHash.String()))) + traceFile, err := os.Create(filepath.Join(baseDir, fmt.Sprintf("trace-%d-%v.jsonl", txIndex, txHash.String()))) if err != nil { return nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err)) } @@ -108,7 +108,7 @@ func Transition(ctx *cli.Context) error { config = []byte(ctx.String(TraceTracerConfigFlag.Name)) } getTracer = func(txIndex int, txHash common.Hash) (vm.EVMLogger, error) { - traceFile, err := os.Create(path.Join(baseDir, fmt.Sprintf("trace-%d-%v.json", txIndex, txHash.String()))) + traceFile, err := os.Create(filepath.Join(baseDir, fmt.Sprintf("trace-%d-%v.json", txIndex, txHash.String()))) if err != nil { return nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err)) } @@ -135,7 +135,7 @@ func Transition(ctx *cli.Context) error { if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) } } if allocStr != stdinSelector { @@ -302,7 +302,7 @@ func saveFile(baseDir, filename string, data interface{}) error { if err != nil { return NewError(ErrorJson, fmt.Errorf("failed marshalling output: %v", err)) } - location := path.Join(baseDir, filename) + location := filepath.Join(baseDir, filename) if err = os.WriteFile(location, b, 0644); err != nil { return NewError(ErrorIO, fmt.Errorf("failed writing output: %v", err)) } diff --git a/cmd/evm/internal/t8ntool/tx_iterator.go b/cmd/evm/internal/t8ntool/tx_iterator.go index 8f28dc70223b..046f62314dae 100644 --- a/cmd/evm/internal/t8ntool/tx_iterator.go +++ b/cmd/evm/internal/t8ntool/tx_iterator.go @@ -127,7 +127,7 @@ func loadTransactions(txStr string, inputData *input, env stEnv, chainConfig *pa return newRlpTxIterator(body), nil } if err := json.Unmarshal(data, &txsWithKeys); err != nil { - return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err)) + return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshalling txs-file: %v", err)) } } else { if len(inputData.TxRlp) > 0 { diff --git a/cmd/evm/internal/t8ntool/utils.go b/cmd/evm/internal/t8ntool/utils.go index 8ec38c7618de..42f5471e7b24 100644 --- a/cmd/evm/internal/t8ntool/utils.go +++ b/cmd/evm/internal/t8ntool/utils.go @@ -33,7 +33,7 @@ func readFile(path, desc string, dest interface{}) error { defer inFile.Close() decoder := json.NewDecoder(inFile) if err := decoder.Decode(dest); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling %s file: %v", desc, err)) + return NewError(ErrorJson, fmt.Errorf("failed unmarshalling %s file: %v", desc, err)) } return nil } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index d333c175599d..17aab678768d 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -444,7 +444,7 @@ func importHistory(ctx *cli.Context) error { return fmt.Errorf("no era1 files found in %s", dir) } if len(networks) > 1 { - return fmt.Errorf("multiple networks found, use a network flag to specify desired network") + return errors.New("multiple networks found, use a network flag to specify desired network") } network = networks[0] } @@ -514,13 +514,10 @@ func importPreimages(ctx *cli.Context) error { return nil } -func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) { - db := utils.MakeChainDatabase(ctx, stack, true) - defer db.Close() - +func parseDumpConfig(ctx *cli.Context, stack *node.Node, db ethdb.Database) (*state.DumpConfig, common.Hash, error) { var header *types.Header if ctx.NArg() > 1 { - return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) + return nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) } if ctx.NArg() == 1 { arg := ctx.Args().First() @@ -529,17 +526,17 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth if number := rawdb.ReadHeaderNumber(db, hash); number != nil { header = rawdb.ReadHeader(db, hash, *number) } else { - return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash) + return nil, common.Hash{}, fmt.Errorf("block %x not found", hash) } } else { number, err := strconv.ParseUint(arg, 10, 64) if err != nil { - return nil, nil, common.Hash{}, err + return nil, common.Hash{}, err } if hash := rawdb.ReadCanonicalHash(db, number); hash != (common.Hash{}) { header = rawdb.ReadHeader(db, hash, number) } else { - return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) + return nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) } } } else { @@ -547,7 +544,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth header = rawdb.ReadHeadHeader(db) } if header == nil { - return nil, nil, common.Hash{}, errors.New("no head block found") + return nil, common.Hash{}, errors.New("no head block found") } startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name)) var start common.Hash @@ -559,7 +556,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth start = crypto.Keccak256Hash(startArg) log.Info("Converting start-address to hash", "address", common.BytesToAddress(startArg), "hash", start.Hex()) default: - return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) + return nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) } var conf = &state.DumpConfig{ SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name), @@ -571,14 +568,17 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(), "skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage, "start", hexutil.Encode(conf.Start), "limit", conf.Max) - return conf, db, header.Root, nil + return conf, header.Root, nil } func dump(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - conf, db, root, err := parseDumpConfig(ctx, stack) + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + conf, root, err := parseDumpConfig(ctx, stack, db) if err != nil { return err } diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 5f52f1df5442..cf4cdef76ce1 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/accounts/usbwallet" + "github.com/ethereum/go-ethereum/beacon/blsync" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -221,6 +222,8 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { } catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon) stack.RegisterLifecycle(simBeacon) + } else if ctx.IsSet(utils.BeaconApiFlag.Name) { + stack.RegisterLifecycle(catalyst.NewBlsync(blsync.NewClient(ctx), eth)) } else { err := catalyst.Register(stack, eth) if err != nil { @@ -264,7 +267,7 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { cfg.Metrics.Enabled = ctx.Bool(utils.MetricsEnabledFlag.Name) } if ctx.IsSet(utils.MetricsEnabledExpensiveFlag.Name) { - cfg.Metrics.EnabledExpensive = ctx.Bool(utils.MetricsEnabledExpensiveFlag.Name) + log.Warn("Expensive metrics are collected by default, please remove this flag", "flag", utils.MetricsEnabledExpensiveFlag.Name) } if ctx.IsSet(utils.MetricsHTTPFlag.Name) { cfg.Metrics.HTTP = ctx.String(utils.MetricsHTTPFlag.Name) diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index ef6ef5f28836..4d6220641703 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -71,7 +71,6 @@ func TestConsoleWelcome(t *testing.T) { Welcome to the Geth JavaScript console! instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}} -coinbase: {{.Etherbase}} at block: 0 ({{niltime}}) datadir: {{.Datadir}} modules: {{apis}} @@ -131,7 +130,6 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) { attach.SetTemplateFunc("goarch", func() string { return runtime.GOARCH }) attach.SetTemplateFunc("gover", runtime.Version) attach.SetTemplateFunc("gethver", func() string { return params.VersionWithCommit("", "") }) - attach.SetTemplateFunc("etherbase", func() string { return geth.Etherbase }) attach.SetTemplateFunc("niltime", func() string { return time.Unix(1548854791, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)") }) @@ -144,7 +142,6 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) { Welcome to the Geth JavaScript console! instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}} -coinbase: {{etherbase}} at block: 0 ({{niltime}}){{if ipc}} datadir: {{datadir}}{{end}} modules: {{apis}} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 2f7d37fdd7e7..d79d23e22687 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console/prompt" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/internal/debug" @@ -116,13 +115,14 @@ var ( utils.DiscoveryPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, - utils.MiningEnabledFlag, + utils.MiningEnabledFlag, // deprecated utils.MinerGasLimitFlag, utils.MinerGasPriceFlag, - utils.MinerEtherbaseFlag, + utils.MinerEtherbaseFlag, // deprecated utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, - utils.MinerNewPayloadTimeout, + utils.MinerPendingFeeRecipientFlag, + utils.MinerNewPayloadTimeoutFlag, // deprecated utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV4Flag, @@ -146,6 +146,14 @@ var ( configFileFlag, utils.LogDebugFlag, utils.LogBacktraceAtFlag, + utils.BeaconApiFlag, + utils.BeaconApiHeaderFlag, + utils.BeaconThresholdFlag, + utils.BeaconNoFilterFlag, + utils.BeaconConfigFlag, + utils.BeaconGenesisRootFlag, + utils.BeaconGenesisTimeFlag, + utils.BeaconCheckpointFlag, }, utils.NetworkFlags, utils.DatabaseFlags) rpcFlags = []cli.Flag{ @@ -421,24 +429,6 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon } }() } - - // Start auxiliary services if enabled - if ctx.Bool(utils.MiningEnabledFlag.Name) { - // Mining only makes sense if a full Ethereum node is running - if ctx.String(utils.SyncModeFlag.Name) == "light" { - utils.Fatalf("Light clients do not support mining") - } - ethBackend, ok := backend.(*eth.EthAPIBackend) - if !ok { - utils.Fatalf("Ethereum service not running") - } - // Set the gas price to the limits from the CLI and start mining - gasprice := flags.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) - ethBackend.TxPool().SetGasTip(gasprice) - if err := ethBackend.StartMining(); err != nil { - utils.Fatalf("Failed to start mining: %v", err) - } - } } // unlockAccounts unlocks any account specifically requested. diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 4284005a0221..192c850868c8 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -36,7 +36,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - cli "github.com/urfave/cli/v2" + "github.com/urfave/cli/v2" ) var ( @@ -541,7 +541,10 @@ func dumpState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - conf, db, root, err := parseDumpConfig(ctx, stack) + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + conf, root, err := parseDumpConfig(ctx, stack, db) if err != nil { return err } diff --git a/cmd/geth/testdata/clique.json b/cmd/geth/testdata/clique.json index b54b4a7d3b72..36f5c3105703 100644 --- a/cmd/geth/testdata/clique.json +++ b/cmd/geth/testdata/clique.json @@ -8,6 +8,7 @@ "byzantiumBlock": 0, "constantinopleBlock": 0, "petersburgBlock": 0, + "terminalTotalDifficultyPassed": true, "clique": { "period": 5, "epoch": 30000 diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 420b063d8ba1..ff3931356e8f 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/gballet/go-verkle" - cli "github.com/urfave/cli/v2" + "github.com/urfave/cli/v2" ) var ( diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 4b5716466556..fc66e11dca96 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -27,7 +27,7 @@ import ( "io" "os" "os/signal" - "path" + "path/filepath" "runtime" "strings" "syscall" @@ -245,13 +245,13 @@ func readList(filename string) ([]string, error) { // starting from genesis. func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error { if chain.CurrentSnapBlock().Number.BitLen() != 0 { - return fmt.Errorf("history import only supported when starting from genesis") + return errors.New("history import only supported when starting from genesis") } entries, err := era.ReadDir(dir, network) if err != nil { return fmt.Errorf("error reading %s: %w", dir, err) } - checksums, err := readList(path.Join(dir, "checksums.txt")) + checksums, err := readList(filepath.Join(dir, "checksums.txt")) if err != nil { return fmt.Errorf("unable to read checksums.txt: %w", err) } @@ -268,7 +268,7 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ ) for i, filename := range entries { err := func() error { - f, err := os.Open(path.Join(dir, filename)) + f, err := os.Open(filepath.Join(dir, filename)) if err != nil { return fmt.Errorf("unable to open era: %w", err) } @@ -425,7 +425,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er ) for i := first; i <= last; i += step { err := func() error { - filename := path.Join(dir, era.Filename(network, int(i/step), common.Hash{})) + filename := filepath.Join(dir, era.Filename(network, int(i/step), common.Hash{})) f, err := os.Create(filename) if err != nil { return fmt.Errorf("could not create era file: %w", err) @@ -458,7 +458,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er return fmt.Errorf("export failed to finalize %d: %w", step/i, err) } // Set correct filename with root. - os.Rename(filename, path.Join(dir, era.Filename(network, int(i/step), root))) + os.Rename(filename, filepath.Join(dir, era.Filename(network, int(i/step), root))) // Compute checksum of entire Era1. if _, err := f.Seek(0, io.SeekStart); err != nil { @@ -481,7 +481,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er } } - os.WriteFile(path.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) + os.WriteFile(filepath.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) log.Info("Exported blockchain to", "dir", dir) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b813e52970d8..b38f33b8dd7f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" + bparams "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/fdlimit" "github.com/ethereum/go-ethereum/core" @@ -281,6 +282,58 @@ var ( Value: ethconfig.Defaults.TransactionHistory, Category: flags.StateCategory, } + // Beacon client light sync settings + BeaconApiFlag = &cli.StringSliceFlag{ + Name: "beacon.api", + Usage: "Beacon node (CL) light client API URL. This flag can be given multiple times.", + Category: flags.BeaconCategory, + } + BeaconApiHeaderFlag = &cli.StringSliceFlag{ + Name: "beacon.api.header", + Usage: "Pass custom HTTP header fields to the emote beacon node API in \"key:value\" format. This flag can be given multiple times.", + Category: flags.BeaconCategory, + } + BeaconThresholdFlag = &cli.IntFlag{ + Name: "beacon.threshold", + Usage: "Beacon sync committee participation threshold", + Value: bparams.SyncCommitteeSupermajority, + Category: flags.BeaconCategory, + } + BeaconNoFilterFlag = &cli.BoolFlag{ + Name: "beacon.nofilter", + Usage: "Disable future slot signature filter", + Category: flags.BeaconCategory, + } + BeaconConfigFlag = &cli.StringFlag{ + Name: "beacon.config", + Usage: "Beacon chain config YAML file", + Category: flags.BeaconCategory, + } + BeaconGenesisRootFlag = &cli.StringFlag{ + Name: "beacon.genesis.gvroot", + Usage: "Beacon chain genesis validators root", + Category: flags.BeaconCategory, + } + BeaconGenesisTimeFlag = &cli.Uint64Flag{ + Name: "beacon.genesis.time", + Usage: "Beacon chain genesis time", + Category: flags.BeaconCategory, + } + BeaconCheckpointFlag = &cli.StringFlag{ + Name: "beacon.checkpoint", + Usage: "Beacon chain weak subjectivity checkpoint block hash", + Category: flags.BeaconCategory, + } + BlsyncApiFlag = &cli.StringFlag{ + Name: "blsync.engine.api", + Usage: "Target EL engine API URL", + Category: flags.BeaconCategory, + } + BlsyncJWTSecretFlag = &cli.StringFlag{ + Name: "blsync.jwtsecret", + Usage: "Path to a JWT secret to use for target engine API endpoint", + Category: flags.BeaconCategory, + } // Transaction pool settings TxPoolLocalsFlag = &cli.StringFlag{ Name: "txpool.locals", @@ -425,11 +478,6 @@ var ( } // Miner settings - MiningEnabledFlag = &cli.BoolFlag{ - Name: "mine", - Usage: "Enable mining", - Category: flags.MinerCategory, - } MinerGasLimitFlag = &cli.Uint64Flag{ Name: "miner.gaslimit", Usage: "Target gas ceiling for mined blocks", @@ -442,11 +490,6 @@ var ( Value: ethconfig.Defaults.Miner.GasPrice, Category: flags.MinerCategory, } - MinerEtherbaseFlag = &cli.StringFlag{ - Name: "miner.etherbase", - Usage: "0x prefixed public address for block mining rewards", - Category: flags.MinerCategory, - } MinerExtraDataFlag = &cli.StringFlag{ Name: "miner.extradata", Usage: "Block extra data set by the miner (default = client version)", @@ -458,10 +501,9 @@ var ( Value: ethconfig.Defaults.Miner.Recommit, Category: flags.MinerCategory, } - MinerNewPayloadTimeout = &cli.DurationFlag{ - Name: "miner.newpayload-timeout", - Usage: "Specify the maximum time allowance for creating a new payload", - Value: ethconfig.Defaults.Miner.NewPayloadTimeout, + MinerPendingFeeRecipientFlag = &cli.StringFlag{ + Name: "miner.pending.feeRecipient", + Usage: "0x prefixed public address for the pending block producer (not used for actual block production)", Category: flags.MinerCategory, } @@ -820,12 +862,6 @@ var ( Usage: "Enable metrics collection and reporting", Category: flags.MetricsCategory, } - MetricsEnabledExpensiveFlag = &cli.BoolFlag{ - Name: "metrics.expensive", - Usage: "Enable expensive metrics collection and reporting", - Category: flags.MetricsCategory, - } - // MetricsHTTPFlag defines the endpoint for a stand-alone metrics HTTP endpoint. // Since the pprof service enables sensitive/vulnerable behavior, this allows a user // to enable a public-OK metrics endpoint without having to worry about ALSO exposing @@ -1268,19 +1304,23 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error // setEtherbase retrieves the etherbase from the directly specified command line flags. func setEtherbase(ctx *cli.Context, cfg *ethconfig.Config) { - if !ctx.IsSet(MinerEtherbaseFlag.Name) { + if ctx.IsSet(MinerEtherbaseFlag.Name) { + log.Warn("Option --miner.etherbase is deprecated as the etherbase is set by the consensus client post-merge") + return + } + if !ctx.IsSet(MinerPendingFeeRecipientFlag.Name) { return } - addr := ctx.String(MinerEtherbaseFlag.Name) + addr := ctx.String(MinerPendingFeeRecipientFlag.Name) if strings.HasPrefix(addr, "0x") || strings.HasPrefix(addr, "0X") { addr = addr[2:] } b, err := hex.DecodeString(addr) if err != nil || len(b) != common.AddressLength { - Fatalf("-%s: invalid etherbase address %q", MinerEtherbaseFlag.Name, addr) + Fatalf("-%s: invalid pending block producer address %q", MinerPendingFeeRecipientFlag.Name, addr) return } - cfg.Miner.Etherbase = common.BytesToAddress(b) + cfg.Miner.PendingFeeRecipient = common.BytesToAddress(b) } // MakePasswordList reads password lines from the file specified by the global --password flag. @@ -1496,6 +1536,9 @@ func setTxPool(ctx *cli.Context, cfg *legacypool.Config) { } func setMiner(ctx *cli.Context, cfg *miner.Config) { + if ctx.Bool(MiningEnabledFlag.Name) { + log.Warn("The flag --mine is deprecated and will be removed") + } if ctx.IsSet(MinerExtraDataFlag.Name) { cfg.ExtraData = []byte(ctx.String(MinerExtraDataFlag.Name)) } @@ -1508,8 +1551,9 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.IsSet(MinerRecommitIntervalFlag.Name) { cfg.Recommit = ctx.Duration(MinerRecommitIntervalFlag.Name) } - if ctx.IsSet(MinerNewPayloadTimeout.Name) { - cfg.NewPayloadTimeout = ctx.Duration(MinerNewPayloadTimeout.Name) + if ctx.IsSet(MinerNewPayloadTimeoutFlag.Name) { + log.Warn("The flag --miner.newpayload-timeout is deprecated and will be removed, please use --miner.recommit") + cfg.Recommit = ctx.Duration(MinerNewPayloadTimeoutFlag.Name) } } @@ -1668,6 +1712,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 { cfg.TransactionHistory = 0 log.Warn("Disabled transaction unindexing for archive node") + + cfg.StateScheme = rawdb.HashScheme + log.Warn("Forcing hash state-scheme for archive mode") } if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 @@ -1783,8 +1830,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // Figure out the dev account address. // setEtherbase has been called above, configuring the miner address from command line flags. - if cfg.Miner.Etherbase != (common.Address{}) { - developer = accounts.Account{Address: cfg.Miner.Etherbase} + if cfg.Miner.PendingFeeRecipient != (common.Address{}) { + developer = accounts.Account{Address: cfg.Miner.PendingFeeRecipient} } else if accs := ks.Accounts(); len(accs) > 0 { developer = ks.Accounts()[0] } else { @@ -1795,7 +1842,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } // Make sure the address is configured as fee recipient, otherwise // the miner will fail to start. - cfg.Miner.Etherbase = developer.Address + cfg.Miner.PendingFeeRecipient = developer.Address if err := ks.Unlock(developer, passphrase); err != nil { Fatalf("Failed to unlock developer account: %v", err) diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 243abd831105..1dfd1a5f8934 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -47,6 +47,9 @@ var DeprecatedFlags = []cli.Flag{ LightNoSyncServeFlag, LogBacktraceAtFlag, LogDebugFlag, + MinerNewPayloadTimeoutFlag, + MinerEtherbaseFlag, + MiningEnabledFlag, } var ( @@ -90,35 +93,35 @@ var ( Name: "light.serve", Usage: "Maximum percentage of time allowed for serving LES requests (deprecated)", Value: ethconfig.Defaults.LightServ, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightIngressFlag = &cli.IntFlag{ Name: "light.ingress", Usage: "Incoming bandwidth limit for serving light clients (deprecated)", Value: ethconfig.Defaults.LightIngress, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightEgressFlag = &cli.IntFlag{ Name: "light.egress", Usage: "Outgoing bandwidth limit for serving light clients (deprecated)", Value: ethconfig.Defaults.LightEgress, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightMaxPeersFlag = &cli.IntFlag{ Name: "light.maxpeers", Usage: "Maximum number of light clients to serve, or light servers to attach to (deprecated)", Value: ethconfig.Defaults.LightPeers, - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightNoPruneFlag = &cli.BoolFlag{ Name: "light.nopruning", Usage: "Disable ancient light chain data pruning (deprecated)", - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } LightNoSyncServeFlag = &cli.BoolFlag{ Name: "light.nosyncserve", Usage: "Enables serving light clients before syncing (deprecated)", - Category: flags.LightCategory, + Category: flags.DeprecatedCategory, } // Deprecated November 2023 LogBacktraceAtFlag = &cli.StringFlag{ @@ -132,6 +135,28 @@ var ( Usage: "Prepends log messages with call-site location (deprecated)", Category: flags.DeprecatedCategory, } + // Deprecated February 2024 + MinerNewPayloadTimeoutFlag = &cli.DurationFlag{ + Name: "miner.newpayload-timeout", + Usage: "Specify the maximum time allowance for creating a new payload (deprecated)", + Value: ethconfig.Defaults.Miner.Recommit, + Category: flags.DeprecatedCategory, + } + MinerEtherbaseFlag = &cli.StringFlag{ + Name: "miner.etherbase", + Usage: "0x prefixed public address for block mining rewards (deprecated)", + Category: flags.DeprecatedCategory, + } + MiningEnabledFlag = &cli.BoolFlag{ + Name: "mine", + Usage: "Enable mining (deprecated)", + Category: flags.DeprecatedCategory, + } + MetricsEnabledExpensiveFlag = &cli.BoolFlag{ + Name: "metrics.expensive", + Usage: "Enable expensive metrics collection and reporting (deprecated)", + Category: flags.DeprecatedCategory, + } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index 9b7f1797d8dd..1d8e48344a2e 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -22,7 +22,7 @@ import ( "io" "math/big" "os" - "path" + "path/filepath" "strings" "testing" @@ -99,7 +99,7 @@ func TestHistoryImportAndExport(t *testing.T) { } // Read checksums. - b, err := os.ReadFile(path.Join(dir, "checksums.txt")) + b, err := os.ReadFile(filepath.Join(dir, "checksums.txt")) if err != nil { t.Fatalf("failed to read checksums: %v", err) } @@ -109,7 +109,7 @@ func TestHistoryImportAndExport(t *testing.T) { entries, _ := era.ReadDir(dir, "mainnet") for i, filename := range entries { func() { - f, err := os.Open(path.Join(dir, filename)) + f, err := os.Open(filepath.Join(dir, filename)) if err != nil { t.Fatalf("error opening era file: %v", err) } diff --git a/common/bitutil/bitutil.go b/common/bitutil/bitutil.go index cd3e72169fc5..a18a6d18eed8 100644 --- a/common/bitutil/bitutil.go +++ b/common/bitutil/bitutil.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Adapted from: https://golang.org/src/crypto/cipher/xor.go +// Adapted from: https://go.dev/src/crypto/subtle/xor_generic.go // Package bitutil implements fast bitwise operations. package bitutil diff --git a/common/bitutil/bitutil_test.go b/common/bitutil/bitutil_test.go index 307bf731f765..12f3fe24a6c9 100644 --- a/common/bitutil/bitutil_test.go +++ b/common/bitutil/bitutil_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Adapted from: https://golang.org/src/crypto/cipher/xor_test.go +// Adapted from: https://go.dev/src/crypto/subtle/xor_test.go package bitutil diff --git a/common/math/big.go b/common/math/big.go index 013c0ba4b66f..721b297c9c4d 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -224,7 +224,7 @@ func ReadBits(bigint *big.Int, buf []byte) { } } -// U256 encodes as a 256 bit two's complement number. This operation is destructive. +// U256 encodes x as a 256 bit two's complement number. This operation is destructive. func U256(x *big.Int) *big.Int { return x.And(x, tt256m1) } @@ -255,14 +255,15 @@ func S256(x *big.Int) *big.Int { // // Courtesy @karalabe and @chfast func Exp(base, exponent *big.Int) *big.Int { + copyBase := new(big.Int).Set(base) result := big.NewInt(1) for _, word := range exponent.Bits() { for i := 0; i < wordBits; i++ { if word&1 == 1 { - U256(result.Mul(result, base)) + U256(result.Mul(result, copyBase)) } - U256(base.Mul(base, base)) + U256(copyBase.Mul(copyBase, copyBase)) word >>= 1 } } diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index c693189ea5ef..59f0e96ebe3e 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - lru "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" diff --git a/consensus/consensus.go b/consensus/consensus.go index 3a2c2d222916..5cc052cb0fea 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -119,11 +119,3 @@ type Engine interface { // Close terminates any background threads maintained by the consensus engine. Close() error } - -// PoW is a consensus engine based on proof-of-work. -type PoW interface { - Engine - - // Hashrate returns the current mining hashrate of a PoW consensus engine. - Hashrate() float64 -} diff --git a/consensus/merger.go b/consensus/merger.go deleted file mode 100644 index ffbcbf2b8569..000000000000 --- a/consensus/merger.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2021 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package consensus - -import ( - "fmt" - "sync" - - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" -) - -// transitionStatus describes the status of eth1/2 transition. This switch -// between modes is a one-way action which is triggered by corresponding -// consensus-layer message. -type transitionStatus struct { - LeftPoW bool // The flag is set when the first NewHead message received - EnteredPoS bool // The flag is set when the first FinalisedBlock message received -} - -// Merger is an internal help structure used to track the eth1/2 transition status. -// It's a common structure can be used in both full node and light client. -type Merger struct { - db ethdb.KeyValueStore - status transitionStatus - mu sync.RWMutex -} - -// NewMerger creates a new Merger which stores its transition status in the provided db. -func NewMerger(db ethdb.KeyValueStore) *Merger { - var status transitionStatus - blob := rawdb.ReadTransitionStatus(db) - if len(blob) != 0 { - if err := rlp.DecodeBytes(blob, &status); err != nil { - log.Crit("Failed to decode the transition status", "err", err) - } - } - return &Merger{ - db: db, - status: status, - } -} - -// ReachTTD is called whenever the first NewHead message received -// from the consensus-layer. -func (m *Merger) ReachTTD() { - m.mu.Lock() - defer m.mu.Unlock() - - if m.status.LeftPoW { - return - } - m.status = transitionStatus{LeftPoW: true} - blob, err := rlp.EncodeToBytes(m.status) - if err != nil { - panic(fmt.Sprintf("Failed to encode the transition status: %v", err)) - } - rawdb.WriteTransitionStatus(m.db, blob) - log.Info("Left PoW stage") -} - -// FinalizePoS is called whenever the first FinalisedBlock message received -// from the consensus-layer. -func (m *Merger) FinalizePoS() { - m.mu.Lock() - defer m.mu.Unlock() - - if m.status.EnteredPoS { - return - } - m.status = transitionStatus{LeftPoW: true, EnteredPoS: true} - blob, err := rlp.EncodeToBytes(m.status) - if err != nil { - panic(fmt.Sprintf("Failed to encode the transition status: %v", err)) - } - rawdb.WriteTransitionStatus(m.db, blob) - log.Info("Entered PoS stage") -} - -// TDDReached reports whether the chain has left the PoW stage. -func (m *Merger) TDDReached() bool { - m.mu.RLock() - defer m.mu.RUnlock() - - return m.status.LeftPoW -} - -// PoSFinalized reports whether the chain has entered the PoS stage. -func (m *Merger) PoSFinalized() bool { - m.mu.RLock() - defer m.mu.RUnlock() - - return m.status.EnteredPoS -} diff --git a/console/console.go b/console/console.go index cdee53684ecf..5acb4cdccb5b 100644 --- a/console/console.go +++ b/console/console.go @@ -325,9 +325,6 @@ func (c *Console) Welcome() { // Print some generic Geth metadata if res, err := c.jsre.Run(` var message = "instance: " + web3.version.node + "\n"; - try { - message += "coinbase: " + eth.coinbase + "\n"; - } catch (err) {} message += "at block: " + eth.blockNumber + " (" + new Date(1000 * eth.getBlock(eth.blockNumber).timestamp) + ")\n"; try { message += " datadir: " + admin.datadir + "\n"; diff --git a/console/console_test.go b/console/console_test.go index a13be6a99ded..d210a993c2d7 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -96,7 +96,7 @@ func newTester(t *testing.T, confOverride func(*ethconfig.Config)) *tester { ethConf := ðconfig.Config{ Genesis: core.DeveloperGenesisBlock(11_500_000, nil), Miner: miner.Config{ - Etherbase: common.HexToAddress(testAddress), + PendingFeeRecipient: common.HexToAddress(testAddress), }, } if confOverride != nil { @@ -152,8 +152,7 @@ func (env *tester) Close(t *testing.T) { } // Tests that the node lists the correct welcome message, notably that it contains -// the instance name, coinbase account, block number, data directory and supported -// console modules. +// the instance name, block number, data directory and supported console modules. func TestWelcome(t *testing.T) { tester := newTester(t, nil) defer tester.Close(t) @@ -167,14 +166,14 @@ func TestWelcome(t *testing.T) { if want := fmt.Sprintf("instance: %s", testInstance); !strings.Contains(output, want) { t.Fatalf("console output missing instance: have\n%s\nwant also %s", output, want) } - if want := fmt.Sprintf("coinbase: %s", testAddress); !strings.Contains(output, want) { - t.Fatalf("console output missing coinbase: have\n%s\nwant also %s", output, want) - } if want := "at block: 0"; !strings.Contains(output, want) { t.Fatalf("console output missing sync status: have\n%s\nwant also %s", output, want) } if want := fmt.Sprintf("datadir: %s", tester.workspace); !strings.Contains(output, want) { - t.Fatalf("console output missing coinbase: have\n%s\nwant also %s", output, want) + t.Fatalf("console output missing datadir: have\n%s\nwant also %s", output, want) + } + if want := "modules: "; !strings.Contains(output, want) { + t.Fatalf("console output missing modules: have\n%s\nwant also %s", output, want) } } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 385c0afd9d40..2f86b2d751b8 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -94,7 +94,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { preBlocks []*types.Block postBlocks []*types.Block engine consensus.Engine - merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) ) if isClique { var ( @@ -186,11 +185,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { } chain.InsertChain(preBlocks[i : i+1]) } - - // Make the transition - merger.ReachTTD() - merger.FinalizePoS() - // Verify the blocks after the merging for i := 0; i < len(postBlocks); i++ { _, results := engine.VerifyHeaders(chain, []*types.Header{postHeaders[i]}) diff --git a/core/blockchain.go b/core/blockchain.go index b1bbc3d5982d..ba346b010d47 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -50,7 +50,6 @@ import ( "github.com/ethereum/go-ethereum/triedb" "github.com/ethereum/go-ethereum/triedb/hashdb" "github.com/ethereum/go-ethereum/triedb/pathdb" - "golang.org/x/exp/slices" ) var ( @@ -62,26 +61,26 @@ var ( chainInfoGauge = metrics.NewRegisteredGaugeInfo("chain/info", nil) - accountReadTimer = metrics.NewRegisteredTimer("chain/account/reads", nil) - accountHashTimer = metrics.NewRegisteredTimer("chain/account/hashes", nil) - accountUpdateTimer = metrics.NewRegisteredTimer("chain/account/updates", nil) - accountCommitTimer = metrics.NewRegisteredTimer("chain/account/commits", nil) + accountReadTimer = metrics.NewRegisteredResettingTimer("chain/account/reads", nil) + accountHashTimer = metrics.NewRegisteredResettingTimer("chain/account/hashes", nil) + accountUpdateTimer = metrics.NewRegisteredResettingTimer("chain/account/updates", nil) + accountCommitTimer = metrics.NewRegisteredResettingTimer("chain/account/commits", nil) - storageReadTimer = metrics.NewRegisteredTimer("chain/storage/reads", nil) - storageHashTimer = metrics.NewRegisteredTimer("chain/storage/hashes", nil) - storageUpdateTimer = metrics.NewRegisteredTimer("chain/storage/updates", nil) - storageCommitTimer = metrics.NewRegisteredTimer("chain/storage/commits", nil) + storageReadTimer = metrics.NewRegisteredResettingTimer("chain/storage/reads", nil) + storageHashTimer = metrics.NewRegisteredResettingTimer("chain/storage/hashes", nil) + storageUpdateTimer = metrics.NewRegisteredResettingTimer("chain/storage/updates", nil) + storageCommitTimer = metrics.NewRegisteredResettingTimer("chain/storage/commits", nil) - snapshotAccountReadTimer = metrics.NewRegisteredTimer("chain/snapshot/account/reads", nil) - snapshotStorageReadTimer = metrics.NewRegisteredTimer("chain/snapshot/storage/reads", nil) - snapshotCommitTimer = metrics.NewRegisteredTimer("chain/snapshot/commits", nil) + snapshotAccountReadTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/account/reads", nil) + snapshotStorageReadTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/storage/reads", nil) + snapshotCommitTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/commits", nil) - triedbCommitTimer = metrics.NewRegisteredTimer("chain/triedb/commits", nil) + triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) - blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) - blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil) - blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) - blockWriteTimer = metrics.NewRegisteredTimer("chain/write", nil) + blockInsertTimer = metrics.NewRegisteredResettingTimer("chain/inserts", nil) + blockValidationTimer = metrics.NewRegisteredResettingTimer("chain/validation", nil) + blockExecutionTimer = metrics.NewRegisteredResettingTimer("chain/execution", nil) + blockWriteTimer = metrics.NewRegisteredResettingTimer("chain/write", nil) blockReorgMeter = metrics.NewRegisteredMeter("chain/reorg/executes", nil) blockReorgAddMeter = metrics.NewRegisteredMeter("chain/reorg/add", nil) @@ -97,13 +96,11 @@ var ( ) const ( - bodyCacheLimit = 256 - blockCacheLimit = 256 - receiptsCacheLimit = 32 - txLookupCacheLimit = 1024 - maxFutureBlocks = 256 - maxTimeFutureBlocks = 30 - TriesInMemory = 128 + bodyCacheLimit = 256 + blockCacheLimit = 256 + receiptsCacheLimit = 32 + txLookupCacheLimit = 1024 + TriesInMemory = 128 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. // @@ -245,9 +242,6 @@ type BlockChain struct { blockCache *lru.Cache[common.Hash, *types.Block] txLookupCache *lru.Cache[common.Hash, txLookup] - // future blocks are blocks added for later processing - futureBlocks *lru.Cache[common.Hash, *types.Block] - wg sync.WaitGroup quit chan struct{} // shutdown signal, closed in Stop. stopping atomic.Bool // false if chain is running, true when stopped @@ -299,7 +293,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), - futureBlocks: lru.NewCache[common.Hash, *types.Block](maxFutureBlocks), engine: engine, vmConfig: vmConfig, } @@ -449,11 +442,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis } bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root) } - - // Start future block processor. - bc.wg.Add(1) - go bc.updateFutureBlocks() - // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) @@ -628,6 +616,172 @@ func (bc *BlockChain) SetSafe(header *types.Header) { } } +// rewindPathHead implements the logic of rewindHead in the context of hash scheme. +func (bc *BlockChain) rewindHashHead(head *types.Header, root common.Hash) (*types.Header, uint64) { + var ( + limit uint64 // The oldest block that will be searched for this rewinding + beyondRoot = root == common.Hash{} // Flag whether we're beyond the requested root (no root, always true) + pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot point state + rootNumber uint64 // Associated block number of requested root + + start = time.Now() // Timestamp the rewinding is restarted + logged = time.Now() // Timestamp last progress log was printed + ) + // The oldest block to be searched is determined by the pivot block or a constant + // searching threshold. The rationale behind this is as follows: + // + // - Snap sync is selected if the pivot block is available. The earliest available + // state is the pivot block itself, so there is no sense in going further back. + // + // - Full sync is selected if the pivot block does not exist. The hash database + // periodically flushes the state to disk, and the used searching threshold is + // considered sufficient to find a persistent state, even for the testnet. It + // might be not enough for a chain that is nearly empty. In the worst case, + // the entire chain is reset to genesis, and snap sync is re-enabled on top, + // which is still acceptable. + if pivot != nil { + limit = *pivot + } else if head.Number.Uint64() > params.FullImmutabilityThreshold { + limit = head.Number.Uint64() - params.FullImmutabilityThreshold + } + for { + logger := log.Trace + if time.Since(logged) > time.Second*8 { + logged = time.Now() + logger = log.Info + } + logger("Block state missing, rewinding further", "number", head.Number, "hash", head.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) + + // If a root threshold was requested but not yet crossed, check + if !beyondRoot && head.Root == root { + beyondRoot, rootNumber = true, head.Number.Uint64() + } + // If search limit is reached, return the genesis block as the + // new chain head. + if head.Number.Uint64() < limit { + log.Info("Rewinding limit reached, resetting to genesis", "number", head.Number, "hash", head.Hash(), "limit", limit) + return bc.genesisBlock.Header(), rootNumber + } + // If the associated state is not reachable, continue searching + // backwards until an available state is found. + if !bc.HasState(head.Root) { + // If the chain is gapped in the middle, return the genesis + // block as the new chain head. + parent := bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) + if parent == nil { + log.Error("Missing block in the middle, resetting to genesis", "number", head.Number.Uint64()-1, "hash", head.ParentHash) + return bc.genesisBlock.Header(), rootNumber + } + head = parent + + // If the genesis block is reached, stop searching. + if head.Number.Uint64() == 0 { + log.Info("Genesis block reached", "number", head.Number, "hash", head.Hash()) + return head, rootNumber + } + continue // keep rewinding + } + // Once the available state is found, ensure that the requested root + // has already been crossed. If not, continue rewinding. + if beyondRoot || head.Number.Uint64() == 0 { + log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) + return head, rootNumber + } + log.Debug("Skipping block with threshold state", "number", head.Number, "hash", head.Hash(), "root", head.Root) + head = bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) // Keep rewinding + } +} + +// rewindPathHead implements the logic of rewindHead in the context of path scheme. +func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*types.Header, uint64) { + var ( + pivot = rawdb.ReadLastPivotNumber(bc.db) // Associated block number of pivot block + rootNumber uint64 // Associated block number of requested root + + // BeyondRoot represents whether the requested root is already + // crossed. The flag value is set to true if the root is empty. + beyondRoot = root == common.Hash{} + + // noState represents if the target state requested for search + // is unavailable and impossible to be recovered. + noState = !bc.HasState(root) && !bc.stateRecoverable(root) + + start = time.Now() // Timestamp the rewinding is restarted + logged = time.Now() // Timestamp last progress log was printed + ) + // Rewind the head block tag until an available state is found. + for { + logger := log.Trace + if time.Since(logged) > time.Second*8 { + logged = time.Now() + logger = log.Info + } + logger("Block state missing, rewinding further", "number", head.Number, "hash", head.Hash(), "elapsed", common.PrettyDuration(time.Since(start))) + + // If a root threshold was requested but not yet crossed, check + if !beyondRoot && head.Root == root { + beyondRoot, rootNumber = true, head.Number.Uint64() + } + // If the root threshold hasn't been crossed but the available + // state is reached, quickly determine if the target state is + // possible to be reached or not. + if !beyondRoot && noState && bc.HasState(head.Root) { + beyondRoot = true + log.Info("Disable the search for unattainable state", "root", root) + } + // Check if the associated state is available or recoverable if + // the requested root has already been crossed. + if beyondRoot && (bc.HasState(head.Root) || bc.stateRecoverable(head.Root)) { + break + } + // If pivot block is reached, return the genesis block as the + // new chain head. Theoretically there must be a persistent + // state before or at the pivot block, prevent endless rewinding + // towards the genesis just in case. + if pivot != nil && *pivot >= head.Number.Uint64() { + log.Info("Pivot block reached, resetting to genesis", "number", head.Number, "hash", head.Hash()) + return bc.genesisBlock.Header(), rootNumber + } + // If the chain is gapped in the middle, return the genesis + // block as the new chain head + parent := bc.GetHeader(head.ParentHash, head.Number.Uint64()-1) // Keep rewinding + if parent == nil { + log.Error("Missing block in the middle, resetting to genesis", "number", head.Number.Uint64()-1, "hash", head.ParentHash) + return bc.genesisBlock.Header(), rootNumber + } + head = parent + + // If the genesis block is reached, stop searching. + if head.Number.Uint64() == 0 { + log.Info("Genesis block reached", "number", head.Number, "hash", head.Hash()) + return head, rootNumber + } + } + // Recover if the target state if it's not available yet. + if !bc.HasState(head.Root) { + if err := bc.triedb.Recover(head.Root); err != nil { + log.Crit("Failed to rollback state", "err", err) + } + } + log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) + return head, rootNumber +} + +// rewindHead searches the available states in the database and returns the associated +// block as the new head block. +// +// If the given root is not empty, then the rewind should attempt to pass the specified +// state root and return the associated block number as well. If the root, typically +// representing the state corresponding to snapshot disk layer, is deemed impassable, +// then block number zero is returned, indicating that snapshot recovery is disabled +// and the whole snapshot should be auto-generated in case of head mismatch. +func (bc *BlockChain) rewindHead(head *types.Header, root common.Hash) (*types.Header, uint64) { + if bc.triedb.Scheme() == rawdb.PathScheme { + return bc.rewindPathHead(head, root) + } + return bc.rewindHashHead(head, root) +} + // setHeadBeyondRoot rewinds the local chain to a new head with the extra condition // that the rewind must pass the specified state root. This method is meant to be // used when rewinding with snapshots enabled to ensure that we go back further than @@ -646,79 +800,40 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha } defer bc.chainmu.Unlock() - // Track the block number of the requested root hash - var rootNumber uint64 // (no root == always 0) - - // Retrieve the last pivot block to short circuit rollbacks beyond it and the - // current freezer limit to start nuking id underflown - pivot := rawdb.ReadLastPivotNumber(bc.db) - frozen, _ := bc.db.Ancients() + var ( + // Track the block number of the requested root hash + rootNumber uint64 // (no root == always 0) + // Retrieve the last pivot block to short circuit rollbacks beyond it + // and the current freezer limit to start nuking it's underflown. + pivot = rawdb.ReadLastPivotNumber(bc.db) + ) updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (*types.Header, bool) { // Rewind the blockchain, ensuring we don't end up with a stateless head // block. Note, depth equality is permitted to allow using SetHead as a // chain reparation mechanism without deleting any data! if currentBlock := bc.CurrentBlock(); currentBlock != nil && header.Number.Uint64() <= currentBlock.Number.Uint64() { - newHeadBlock := bc.GetBlock(header.Hash(), header.Number.Uint64()) - if newHeadBlock == nil { - log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash()) - newHeadBlock = bc.genesisBlock - } else { - // Block exists. Keep rewinding until either we find one with state - // or until we exceed the optional threshold root hash - beyondRoot := (root == common.Hash{}) // Flag whether we're beyond the requested root (no root, always true) - - for { - // If a root threshold was requested but not yet crossed, check - if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root { - beyondRoot, rootNumber = true, newHeadBlock.NumberU64() - } - if !bc.HasState(newHeadBlock.Root()) && !bc.stateRecoverable(newHeadBlock.Root()) { - log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) - if pivot == nil || newHeadBlock.NumberU64() > *pivot { - parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) - if parent != nil { - newHeadBlock = parent - continue - } - log.Error("Missing block in the middle, aiming genesis", "number", newHeadBlock.NumberU64()-1, "hash", newHeadBlock.ParentHash()) - newHeadBlock = bc.genesisBlock - } else { - log.Trace("Rewind passed pivot, aiming genesis", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "pivot", *pivot) - newHeadBlock = bc.genesisBlock - } - } - if beyondRoot || newHeadBlock.NumberU64() == 0 { - if !bc.HasState(newHeadBlock.Root()) && bc.stateRecoverable(newHeadBlock.Root()) { - // Rewind to a block with recoverable state. If the state is - // missing, run the state recovery here. - if err := bc.triedb.Recover(newHeadBlock.Root()); err != nil { - log.Crit("Failed to rollback state", "err", err) // Shouldn't happen - } - log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) - } - break - } - log.Debug("Skipping block with threshold state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "root", newHeadBlock.Root()) - newHeadBlock = bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) // Keep rewinding - } - } + var newHeadBlock *types.Header + newHeadBlock, rootNumber = bc.rewindHead(header, root) rawdb.WriteHeadBlockHash(db, newHeadBlock.Hash()) // Degrade the chain markers if they are explicitly reverted. // In theory we should update all in-memory markers in the // last step, however the direction of SetHead is from high // to low, so it's safe to update in-memory markers directly. - bc.currentBlock.Store(newHeadBlock.Header()) - headBlockGauge.Update(int64(newHeadBlock.NumberU64())) + bc.currentBlock.Store(newHeadBlock) + headBlockGauge.Update(int64(newHeadBlock.Number.Uint64())) // The head state is missing, which is only possible in the path-based // scheme. This situation occurs when the chain head is rewound below // the pivot point. In this scenario, there is no possible recovery // approach except for rerunning a snap sync. Do nothing here until the // state syncer picks it up. - if !bc.HasState(newHeadBlock.Root()) { - log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number(), "hash", newHeadBlock.Hash()) + if !bc.HasState(newHeadBlock.Root) { + if newHeadBlock.Number.Uint64() != 0 { + log.Crit("Chain is stateless at a non-genesis block") + } + log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number, "hash", newHeadBlock.Hash()) } } // Rewind the snap block in a simpleton way to the target head @@ -745,6 +860,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha // intent afterwards is full block importing, delete the chain segment // between the stateful-block and the sethead target. var wipe bool + frozen, _ := bc.db.Ancients() if headNumber+1 < frozen { wipe = pivot == nil || headNumber >= *pivot } @@ -794,7 +910,6 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha bc.receiptsCache.Purge() bc.blockCache.Purge() bc.txLookupCache.Purge() - bc.futureBlocks.Purge() // Clear safe block, finalized block if needed if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.Number.Uint64() { @@ -1048,24 +1163,6 @@ func (bc *BlockChain) insertStopped() bool { return bc.procInterrupt.Load() } -func (bc *BlockChain) procFutureBlocks() { - blocks := make([]*types.Block, 0, bc.futureBlocks.Len()) - for _, hash := range bc.futureBlocks.Keys() { - if block, exist := bc.futureBlocks.Peek(hash); exist { - blocks = append(blocks, block) - } - } - if len(blocks) > 0 { - slices.SortFunc(blocks, func(a, b *types.Block) int { - return a.Number().Cmp(b.Number()) - }) - // Insert one by one as chain insertion needs contiguous ancestry between blocks - for i := range blocks { - bc.InsertChain(blocks[i : i+1]) - } - } -} - // WriteStatus status of write type WriteStatus byte @@ -1466,8 +1563,6 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types if status == CanonStatTy { bc.writeHeadBlock(block) } - bc.futureBlocks.Remove(block.Hash()) - if status == CanonStatTy { bc.chainFeed.Send(ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) if len(logs) > 0 { @@ -1487,25 +1582,6 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types return status, nil } -// addFutureBlock checks if the block is within the max allowed window to get -// accepted for future processing, and returns an error if the block is too far -// ahead and was not added. -// -// TODO after the transition, the future block shouldn't be kept. Because -// it's not checked in the Geth side anymore. -func (bc *BlockChain) addFutureBlock(block *types.Block) error { - max := uint64(time.Now().Unix() + maxTimeFutureBlocks) - if block.Time() > max { - return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max) - } - if block.Difficulty().Cmp(common.Big0) == 0 { - // Never add PoS blocks into the future queue - return nil - } - bc.futureBlocks.Add(block.Hash(), block) - return nil -} - // InsertChain attempts to insert the given batch of blocks in to the canonical // chain or, otherwise, create a fork. If an error is returned it will return // the index number of the failing block as well an error describing what went @@ -1643,26 +1719,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) _, err := bc.recoverAncestors(block) return it.index, err } - // First block is future, shove it (and all children) to the future queue (unknown ancestor) - case errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash())): - for block != nil && (it.index == 0 || errors.Is(err, consensus.ErrUnknownAncestor)) { - log.Debug("Future block, postponing import", "number", block.Number(), "hash", block.Hash()) - if err := bc.addFutureBlock(block); err != nil { - return it.index, err - } - block, err = it.next() - } - stats.queued += it.processed() - stats.ignored += it.remaining() - - // If there are any still remaining, mark as ignored - return it.index, err - // Some other error(except ErrKnownBlock) occurred, abort. // ErrKnownBlock is allowed here since some known blocks // still need re-execution to generate snapshots that are missing case err != nil && !errors.Is(err, ErrKnownBlock): - bc.futureBlocks.Remove(block.Hash()) stats.ignored += len(it.chain) bc.reportBlock(block, nil, err) return it.index, err @@ -1867,23 +1927,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) "root", block.Root()) } } - - // Any blocks remaining here? The only ones we care about are the future ones - if block != nil && errors.Is(err, consensus.ErrFutureBlock) { - if err := bc.addFutureBlock(block); err != nil { - return it.index, err - } - block, err = it.next() - - for ; block != nil && errors.Is(err, consensus.ErrUnknownAncestor); block, err = it.next() { - if err := bc.addFutureBlock(block); err != nil { - return it.index, err - } - stats.queued++ - } - } stats.ignored += it.remaining() - return it.index, err } @@ -2334,20 +2378,6 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) { return head.Hash(), nil } -func (bc *BlockChain) updateFutureBlocks() { - futureTimer := time.NewTicker(5 * time.Second) - defer futureTimer.Stop() - defer bc.wg.Done() - for { - select { - case <-futureTimer.C: - bc.procFutureBlocks() - case <-bc.quit: - return - } - } -} - // skipBlock returns 'true', if the block being imported can be skipped over, meaning // that the block does not need to be processed but can be considered already fully 'done'. func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool { diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index 9bf662b6b710..c7c4c4bfea8b 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -179,8 +179,3 @@ func (it *insertIterator) first() *types.Block { func (it *insertIterator) remaining() int { return len(it.chain) - it.index } - -// processed returns the number of processed blocks. -func (it *insertIterator) processed() int { - return it.index + 1 -} diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index b2df39d17bb8..a4761f337b85 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -22,7 +22,7 @@ package core import ( "math/big" - "path" + "path/filepath" "testing" "time" @@ -1757,12 +1757,12 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) { // It's hard to follow the test case, visualize the input - //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // fmt.Println(tt.dump(true)) // Create a temporary persistent database datadir := t.TempDir() - ancient := path.Join(datadir, "ancient") + ancient := filepath.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, @@ -1830,10 +1830,14 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s } // Force run a freeze cycle type freezer interface { - Freeze(threshold uint64) error + Freeze() error Ancients() (uint64, error) } - db.(freezer).Freeze(tt.freezeThreshold) + if tt.freezeThreshold < uint64(tt.canonicalBlocks) { + final := uint64(tt.canonicalBlocks) - tt.freezeThreshold + chain.SetFinalized(canonblocks[int(final)-1].Header()) + } + db.(freezer).Freeze() // Set the simulated pivot block if tt.pivotBlock != nil { @@ -1908,7 +1912,7 @@ func testIssue23496(t *testing.T, scheme string) { // Create a temporary persistent database datadir := t.TempDir() - ancient := path.Join(datadir, "ancient") + ancient := filepath.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 1504c74e0ef3..b96ee12c9952 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -2044,10 +2044,14 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme // Force run a freeze cycle type freezer interface { - Freeze(threshold uint64) error + Freeze() error Ancients() (uint64, error) } - db.(freezer).Freeze(tt.freezeThreshold) + if tt.freezeThreshold < uint64(tt.canonicalBlocks) { + final := uint64(tt.canonicalBlocks) - tt.freezeThreshold + chain.SetFinalized(canonblocks[int(final)-1].Header()) + } + db.(freezer).Freeze() // Set the simulated pivot block if tt.pivotBlock != nil { diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index dd012c430c4d..80f8035df151 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -24,7 +24,7 @@ import ( "fmt" "math/big" "os" - "path" + "path/filepath" "strings" "testing" "time" @@ -63,7 +63,7 @@ type snapshotTestBasic struct { func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) { // Create a temporary persistent database datadir := t.TempDir() - ancient := path.Join(datadir, "ancient") + ancient := filepath.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 876d662f74d8..4fa759129c9a 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -2129,7 +2129,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate a canonical chain to act as the main dataset chainConfig := *params.TestChainConfig var ( - merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) engine = beacon.New(ethash.NewFaker()) key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key.PublicKey) @@ -2153,8 +2152,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Activate the transition since genesis if required if mergePoint == 0 { mergeBlock = 0 - merger.ReachTTD() - merger.FinalizePoS() // Set the terminal total difficulty in the config gspec.Config.TerminalTotalDifficulty = big.NewInt(0) @@ -2189,8 +2186,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Activate the transition in the middle of the chain if mergePoint == 1 { - merger.ReachTTD() - merger.FinalizePoS() // Set the terminal total difficulty in the config ttd := big.NewInt(int64(len(blocks))) ttd.Mul(ttd, params.GenesisDifficulty) diff --git a/core/genesis.go b/core/genesis.go index 54570ac61e4c..3f1fde8dfca4 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -45,7 +45,7 @@ import ( var errGenesisNoConfig = errors.New("genesis has no chain configuration") -// Deprecated: use types.GenesisAccount instead. +// Deprecated: use types.Account instead. type GenesisAccount = types.Account // Deprecated: use types.GenesisAlloc instead. diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index ea3367db3606..e34b24fd7661 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -315,7 +315,7 @@ func ReadStateScheme(db ethdb.Reader) string { // the stored state. // // - If the provided scheme is none, use the scheme consistent with persistent -// state, or fallback to hash-based scheme if state is empty. +// state, or fallback to path-based scheme if state is empty. // // - If the provided scheme is hash, use hash-based scheme or error out if not // compatible with persistent state scheme. @@ -329,10 +329,8 @@ func ParseStateScheme(provided string, disk ethdb.Database) (string, error) { stored := ReadStateScheme(disk) if provided == "" { if stored == "" { - // use default scheme for empty database, flip it when - // path mode is chosen as default - log.Info("State schema set to default", "scheme", "hash") - return HashScheme, nil + log.Info("State schema set to default", "scheme", "path") + return PathScheme, nil // use default scheme for empty database } log.Info("State scheme set to already existing", "scheme", stored) return stored, nil // reuse scheme of persistent scheme diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index bb2c409dbbd5..d8214874bdb8 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -17,9 +17,9 @@ package rawdb import ( + "errors" "fmt" "sync" - "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -43,8 +43,6 @@ const ( // The background thread will keep moving ancient chain segments from key-value // database to flat files for saving space on live database. type chainFreezer struct { - threshold atomic.Uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests) - *Freezer quit chan struct{} wg sync.WaitGroup @@ -57,13 +55,11 @@ func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFre if err != nil { return nil, err } - cf := chainFreezer{ + return &chainFreezer{ Freezer: freezer, quit: make(chan struct{}), trigger: make(chan chan struct{}), - } - cf.threshold.Store(params.FullImmutabilityThreshold) - return &cf, nil + }, nil } // Close closes the chain freezer instance and terminates the background thread. @@ -77,6 +73,57 @@ func (f *chainFreezer) Close() error { return f.Freezer.Close() } +// readHeadNumber returns the number of chain head block. 0 is returned if the +// block is unknown or not available yet. +func (f *chainFreezer) readHeadNumber(db ethdb.KeyValueReader) uint64 { + hash := ReadHeadBlockHash(db) + if hash == (common.Hash{}) { + log.Error("Head block is not reachable") + return 0 + } + number := ReadHeaderNumber(db, hash) + if number == nil { + log.Error("Number of head block is missing") + return 0 + } + return *number +} + +// readFinalizedNumber returns the number of finalized block. 0 is returned +// if the block is unknown or not available yet. +func (f *chainFreezer) readFinalizedNumber(db ethdb.KeyValueReader) uint64 { + hash := ReadFinalizedBlockHash(db) + if hash == (common.Hash{}) { + return 0 + } + number := ReadHeaderNumber(db, hash) + if number == nil { + log.Error("Number of finalized block is missing") + return 0 + } + return *number +} + +// freezeThreshold returns the threshold for chain freezing. It's determined +// by formula: max(finality, HEAD-params.FullImmutabilityThreshold). +func (f *chainFreezer) freezeThreshold(db ethdb.KeyValueReader) (uint64, error) { + var ( + head = f.readHeadNumber(db) + final = f.readFinalizedNumber(db) + headLimit uint64 + ) + if head > params.FullImmutabilityThreshold { + headLimit = head - params.FullImmutabilityThreshold + } + if final == 0 && headLimit == 0 { + return 0, errors.New("freezing threshold is not available") + } + if final > headLimit { + return final, nil + } + return headLimit, nil +} + // freeze is a background thread that periodically checks the blockchain for any // import progress and moves ancient data from the fast database into the freezer. // @@ -114,60 +161,39 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { return } } - // Retrieve the freezing threshold. - hash := ReadHeadBlockHash(nfdb) - if hash == (common.Hash{}) { - log.Debug("Current full block hash unavailable") // new chain, empty database + threshold, err := f.freezeThreshold(nfdb) + if err != nil { backoff = true + log.Debug("Current full block not old enough to freeze", "err", err) continue } - number := ReadHeaderNumber(nfdb, hash) - threshold := f.threshold.Load() frozen := f.frozen.Load() - switch { - case number == nil: - log.Error("Current full block number unavailable", "hash", hash) - backoff = true - continue - case *number < threshold: - log.Debug("Current full block not old enough to freeze", "number", *number, "hash", hash, "delay", threshold) - backoff = true - continue - - case *number-threshold <= frozen: - log.Debug("Ancient blocks frozen already", "number", *number, "hash", hash, "frozen", frozen) + // Short circuit if the blocks below threshold are already frozen. + if frozen != 0 && frozen-1 >= threshold { backoff = true + log.Debug("Ancient blocks frozen already", "threshold", threshold, "frozen", frozen) continue } - head := ReadHeader(nfdb, hash, *number) - if head == nil { - log.Error("Current full block unavailable", "number", *number, "hash", hash) - backoff = true - continue - } - // Seems we have data ready to be frozen, process in usable batches var ( - start = time.Now() - first, _ = f.Ancients() - limit = *number - threshold + start = time.Now() + first = frozen // the first block to freeze + last = threshold // the last block to freeze ) - if limit-first > freezerBatchLimit { - limit = first + freezerBatchLimit + if last-first+1 > freezerBatchLimit { + last = freezerBatchLimit + first - 1 } - ancients, err := f.freezeRange(nfdb, first, limit) + ancients, err := f.freezeRange(nfdb, first, last) if err != nil { log.Error("Error in block freeze operation", "err", err) backoff = true continue } - // Batch of blocks have been frozen, flush them before wiping from leveldb if err := f.Sync(); err != nil { log.Crit("Failed to flush frozen tables", "err", err) } - // Wipe out all data from the active database batch := db.NewBatch() for i := 0; i < len(ancients); i++ { @@ -250,8 +276,11 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { } } +// freezeRange moves a batch of chain segments from the fast database to the freezer. +// The parameters (number, limit) specify the relevant block range, both of which +// are included. func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes []common.Hash, err error) { - hashes = make([]common.Hash, 0, limit-number) + hashes = make([]common.Hash, 0, limit-number+1) _, err = f.ModifyAncients(func(op ethdb.AncientWriteOp) error { for ; number <= limit; number++ { @@ -293,11 +322,9 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash if err := op.AppendRaw(ChainFreezerDifficultyTable, number, td); err != nil { return fmt.Errorf("can't write td to Freezer: %v", err) } - hashes = append(hashes, hash) } return nil }) - return hashes, err } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 27a9ec7412ca..9cab30bfcd1f 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -66,16 +66,10 @@ func (frdb *freezerdb) Close() error { // Freeze is a helper method used for external testing to trigger and block until // a freeze cycle completes, without having to sleep for a minute to trigger the // automatic background run. -func (frdb *freezerdb) Freeze(threshold uint64) error { +func (frdb *freezerdb) Freeze() error { if frdb.AncientStore.(*chainFreezer).readonly { return errReadOnly } - // Set the freezer threshold to a temporary value - defer func(old uint64) { - frdb.AncientStore.(*chainFreezer).threshold.Store(old) - }(frdb.AncientStore.(*chainFreezer).threshold.Load()) - frdb.AncientStore.(*chainFreezer).threshold.Store(threshold) - // Trigger a freeze cycle and block until it's done trigger := make(chan struct{}, 1) frdb.AncientStore.(*chainFreezer).trigger <- trigger diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index b4bd6a382a86..2a156638903d 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -24,6 +24,7 @@ import ( "math/rand" "os" "path" + "path/filepath" "sync" "testing" @@ -393,7 +394,7 @@ func TestRenameWindows(t *testing.T) { dir2 := t.TempDir() // Create file in dir1 and fill with data - f, err := os.Create(path.Join(dir1, fname)) + f, err := os.Create(filepath.Join(dir1, fname)) if err != nil { t.Fatal(err) } diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 11cf5b40fef6..dbf010be0ca8 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -113,8 +113,8 @@ var ( skeletonHeaderPrefix = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header // Path-based storage scheme of merkle patricia trie. - trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node - trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node + TrieNodeAccountPrefix = []byte("A") // TrieNodeAccountPrefix + hexPath -> trie node + TrieNodeStoragePrefix = []byte("O") // TrieNodeStoragePrefix + accountHash + hexPath -> trie node stateIDPrefix = []byte("L") // stateIDPrefix + state root -> state id PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage @@ -265,15 +265,15 @@ func stateIDKey(root common.Hash) []byte { return append(stateIDPrefix, root.Bytes()...) } -// accountTrieNodeKey = trieNodeAccountPrefix + nodePath. +// accountTrieNodeKey = TrieNodeAccountPrefix + nodePath. func accountTrieNodeKey(path []byte) []byte { - return append(trieNodeAccountPrefix, path...) + return append(TrieNodeAccountPrefix, path...) } -// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath. +// storageTrieNodeKey = TrieNodeStoragePrefix + accountHash + nodePath. func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { - buf := make([]byte, len(trieNodeStoragePrefix)+common.HashLength+len(path)) - n := copy(buf, trieNodeStoragePrefix) + buf := make([]byte, len(TrieNodeStoragePrefix)+common.HashLength+len(path)) + n := copy(buf, TrieNodeStoragePrefix) n += copy(buf[n:], accountHash.Bytes()) copy(buf[n:], path) return buf @@ -294,16 +294,16 @@ func IsLegacyTrieNode(key []byte, val []byte) bool { // account trie node in path-based state scheme, and returns the resolved // node path if so. func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) { - if !bytes.HasPrefix(key, trieNodeAccountPrefix) { + if !bytes.HasPrefix(key, TrieNodeAccountPrefix) { return false, nil } // The remaining key should only consist a hex node path // whose length is in the range 0 to 64 (64 is excluded // since leaves are always wrapped with shortNode). - if len(key) >= len(trieNodeAccountPrefix)+common.HashLength*2 { + if len(key) >= len(TrieNodeAccountPrefix)+common.HashLength*2 { return false, nil } - return true, key[len(trieNodeAccountPrefix):] + return true, key[len(TrieNodeAccountPrefix):] } // IsAccountTrieNode reports whether a provided database entry is an account @@ -317,20 +317,20 @@ func IsAccountTrieNode(key []byte) bool { // trie node in path-based state scheme, and returns the resolved account hash // and node path if so. func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) { - if !bytes.HasPrefix(key, trieNodeStoragePrefix) { + if !bytes.HasPrefix(key, TrieNodeStoragePrefix) { return false, common.Hash{}, nil } // The remaining key consists of 2 parts: // - 32 bytes account hash // - hex node path whose length is in the range 0 to 64 - if len(key) < len(trieNodeStoragePrefix)+common.HashLength { + if len(key) < len(TrieNodeStoragePrefix)+common.HashLength { return false, common.Hash{}, nil } - if len(key) >= len(trieNodeStoragePrefix)+common.HashLength+common.HashLength*2 { + if len(key) >= len(TrieNodeStoragePrefix)+common.HashLength+common.HashLength*2 { return false, common.Hash{}, nil } - accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength]) - return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:] + accountHash := common.BytesToHash(key[len(TrieNodeStoragePrefix) : len(TrieNodeStoragePrefix)+common.HashLength]) + return true, accountHash, key[len(TrieNodeStoragePrefix)+common.HashLength:] } // IsStorageTrieNode reports whether a provided database entry is a storage diff --git a/core/state/metrics.go b/core/state/metrics.go index 64c651461e73..7447e44dfd1e 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -33,5 +33,4 @@ var ( slotDeletionTimer = metrics.NewRegisteredResettingTimer("state/delete/storage/timer", nil) slotDeletionCount = metrics.NewRegisteredMeter("state/delete/storage/slot", nil) slotDeletionSize = metrics.NewRegisteredMeter("state/delete/storage/size", nil) - slotDeletionSkip = metrics.NewRegisteredGauge("state/delete/storage/skip", nil) ) diff --git a/core/state/state_object.go b/core/state/state_object.go index fc26af68dbe7..6dea68465baa 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/holiman/uint256" @@ -197,9 +196,8 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { if s.db.snap != nil { start := time.Now() enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key.Bytes())) - if metrics.EnabledExpensive { - s.db.SnapshotStorageReads += time.Since(start) - } + s.db.SnapshotStorageReads += time.Since(start) + if len(enc) > 0 { _, content, _, err := rlp.Split(enc) if err != nil { @@ -217,9 +215,8 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { return common.Hash{} } val, err := tr.GetStorage(s.address, key.Bytes()) - if metrics.EnabledExpensive { - s.db.StorageReads += time.Since(start) - } + s.db.StorageReads += time.Since(start) + if err != nil { s.db.setError(err) return common.Hash{} @@ -283,9 +280,8 @@ func (s *stateObject) updateTrie() (Trie, error) { return s.trie, nil } // Track the amount of time wasted on updating the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now()) + // The snapshot storage map for the object var ( storage map[common.Hash][]byte @@ -370,9 +366,8 @@ func (s *stateObject) updateRoot() { return } // Track the amount of time wasted on hashing the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now()) + s.data.Root = tr.Hash() } @@ -386,9 +381,8 @@ func (s *stateObject) commit() (*trienode.NodeSet, error) { return nil, nil } // Track the amount of time wasted on committing the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) + // The trie is currently in an open state and could potentially contain // cached mutations. Call commit to acquire a set of nodes that have been // modified, the set can be nil if nothing to commit. diff --git a/core/state/statedb.go b/core/state/statedb.go index a4b8cf93e2d2..f90b30f3994e 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" @@ -36,12 +35,6 @@ import ( "github.com/holiman/uint256" ) -const ( - // storageDeleteLimit denotes the highest permissible memory allocation - // employed for contract storage deletion. - storageDeleteLimit = 512 * 1024 * 1024 -) - type revision struct { id int journalIndex int @@ -501,9 +494,8 @@ func (s *StateDB) GetTransientState(addr common.Address, key common.Hash) common // updateStateObject writes the given object to the trie. func (s *StateDB) updateStateObject(obj *stateObject) { // Track the amount of time wasted on updating the account from the trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) + // Encode the account and update the account trie addr := obj.Address() if err := s.trie.UpdateAccount(addr, &obj.data); err != nil { @@ -533,9 +525,8 @@ func (s *StateDB) updateStateObject(obj *stateObject) { // deleteStateObject removes the given object from the state trie. func (s *StateDB) deleteStateObject(obj *stateObject) { // Track the amount of time wasted on deleting the account from the trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) + // Delete the account from the trie addr := obj.Address() if err := s.trie.DeleteAccount(addr); err != nil { @@ -567,9 +558,8 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { if s.snap != nil { start := time.Now() acc, err := s.snap.Account(crypto.HashData(s.hasher, addr.Bytes())) - if metrics.EnabledExpensive { - s.SnapshotAccountReads += time.Since(start) - } + s.SnapshotAccountReads += time.Since(start) + if err == nil { if acc == nil { return nil @@ -593,9 +583,8 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { start := time.Now() var err error data, err = s.trie.GetAccount(addr) - if metrics.EnabledExpensive { - s.AccountReads += time.Since(start) - } + s.AccountReads += time.Since(start) + if err != nil { s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %w", addr.Bytes(), err)) return nil @@ -923,9 +912,8 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { s.stateObjectsPending = make(map[common.Address]struct{}) } // Track the amount of time wasted on hashing the account trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) - } + defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) + return s.trie.Hash() } @@ -949,10 +937,10 @@ func (s *StateDB) clearJournalAndRefund() { // of a specific account. It leverages the associated state snapshot for fast // storage iteration and constructs trie node deletion markers by creating // stack trie with iterated slots. -func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { iter, err := s.snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{}) if err != nil { - return false, 0, nil, nil, err + return 0, nil, nil, err } defer iter.Release() @@ -968,40 +956,37 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo }) stack := trie.NewStackTrie(options) for iter.Next() { - if size > storageDeleteLimit { - return true, size, nil, nil, nil - } slot := common.CopyBytes(iter.Slot()) if err := iter.Error(); err != nil { // error might occur after Slot function - return false, 0, nil, nil, err + return 0, nil, nil, err } size += common.StorageSize(common.HashLength + len(slot)) slots[iter.Hash()] = slot if err := stack.Update(iter.Hash().Bytes(), slot); err != nil { - return false, 0, nil, nil, err + return 0, nil, nil, err } } if err := iter.Error(); err != nil { // error might occur during iteration - return false, 0, nil, nil, err + return 0, nil, nil, err } if stack.Hash() != root { - return false, 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) + return 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) } - return false, size, slots, nodes, nil + return size, slots, nodes, nil } // slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage," // employed when the associated state snapshot is not available. It iterates the // storage slots along with all internal trie nodes via trie directly. -func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie) if err != nil { - return false, 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) + return 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) } it, err := tr.NodeIterator(nil) if err != nil { - return false, 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) + return 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) } var ( size common.StorageSize @@ -1009,9 +994,6 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r slots = make(map[common.Hash][]byte) ) for it.Next(true) { - if size > storageDeleteLimit { - return true, size, nil, nil, nil - } if it.Leaf() { slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob()) size += common.StorageSize(common.HashLength + len(it.LeafBlob())) @@ -1024,9 +1006,9 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r nodes.AddNode(it.Path(), trienode.NewDeleted()) } if err := it.Error(); err != nil { - return false, 0, nil, nil, err + return 0, nil, nil, err } - return false, size, slots, nodes, nil + return size, slots, nodes, nil } // deleteStorage is designed to delete the storage trie of a designated account. @@ -1034,41 +1016,37 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r // potentially leading to an out-of-memory panic. The function will make an attempt // to utilize an efficient strategy if the associated state snapshot is reachable; // otherwise, it will resort to a less-efficient approach. -func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { var ( - start = time.Now() - err error - aborted bool - size common.StorageSize - slots map[common.Hash][]byte - nodes *trienode.NodeSet + start = time.Now() + err error + size common.StorageSize + slots map[common.Hash][]byte + nodes *trienode.NodeSet ) // The fast approach can be failed if the snapshot is not fully // generated, or it's internally corrupted. Fallback to the slow // one just in case. if s.snap != nil { - aborted, size, slots, nodes, err = s.fastDeleteStorage(addrHash, root) + size, slots, nodes, err = s.fastDeleteStorage(addrHash, root) } if s.snap == nil || err != nil { - aborted, size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) + size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) } if err != nil { - return false, nil, nil, err + return nil, nil, err } - if metrics.EnabledExpensive { - if aborted { - slotDeletionSkip.Inc(1) - } - n := int64(len(slots)) + // Report the metrics + n := int64(len(slots)) - slotDeletionMaxCount.UpdateIfGt(int64(len(slots))) - slotDeletionMaxSize.UpdateIfGt(int64(size)) + slotDeletionMaxCount.UpdateIfGt(int64(len(slots))) + slotDeletionMaxSize.UpdateIfGt(int64(size)) - slotDeletionTimer.UpdateSince(start) - slotDeletionCount.Mark(n) - slotDeletionSize.Mark(int64(size)) - } - return aborted, slots, nodes, nil + slotDeletionTimer.UpdateSince(start) + slotDeletionCount.Mark(n) + slotDeletionSize.Mark(int64(size)) + + return slots, nodes, nil } // handleDestruction processes all destruction markers and deletes the account @@ -1095,13 +1073,12 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root // // In case (d), **original** account along with its storages should be deleted, // with their values be tracked as original value. -func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.Address]struct{}, error) { +func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) error { // Short circuit if geth is running with hash mode. This procedure can consume // considerable time and storage deletion isn't supported in hash mode, thus // preemptively avoiding unnecessary expenses. - incomplete := make(map[common.Address]struct{}) if s.db.TrieDB().Scheme() == rawdb.HashScheme { - return incomplete, nil + return nil } for addr, prev := range s.stateObjectsDestruct { // The original account was non-existing, and it's marked as destructed @@ -1124,18 +1101,9 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A continue } // Remove storage slots belong to the account. - aborted, slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) + slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) if err != nil { - return nil, fmt.Errorf("failed to delete storage, err: %w", err) - } - // The storage is too huge to handle, skip it but mark as incomplete. - // For case (d), the account is resurrected might with a few slots - // created. In this case, wipe the entire storage state diff because - // of aborted deletion. - if aborted { - incomplete[addr] = struct{}{} - delete(s.storagesOrigin, addr) - continue + return fmt.Errorf("failed to delete storage, err: %w", err) } if s.storagesOrigin[addr] == nil { s.storagesOrigin[addr] = slots @@ -1147,10 +1115,10 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A } } if err := nodes.Merge(set); err != nil { - return nil, err + return err } } - return incomplete, nil + return nil } // Commit writes the state to the underlying in-memory trie database. @@ -1179,8 +1147,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er codeWriter = s.db.DiskDB().NewBatch() ) // Handle all state deletions first - incomplete, err := s.handleDestruction(nodes) - if err != nil { + if err := s.handleDestruction(nodes); err != nil { return common.Hash{}, err } // Handle all state updates afterwards @@ -1217,10 +1184,8 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er } } // Write the account trie changes, measuring the amount of wasted time - var start time.Time - if metrics.EnabledExpensive { - start = time.Now() - } + start := time.Now() + root, set, err := s.trie.Commit(true) if err != nil { return common.Hash{}, err @@ -1232,23 +1197,23 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er } accountTrieNodesUpdated, accountTrieNodesDeleted = set.Size() } - if metrics.EnabledExpensive { - s.AccountCommits += time.Since(start) + // Report the commit metrics + s.AccountCommits += time.Since(start) + + accountUpdatedMeter.Mark(int64(s.AccountUpdated)) + storageUpdatedMeter.Mark(int64(s.StorageUpdated)) + accountDeletedMeter.Mark(int64(s.AccountDeleted)) + storageDeletedMeter.Mark(int64(s.StorageDeleted)) + accountTrieUpdatedMeter.Mark(int64(accountTrieNodesUpdated)) + accountTrieDeletedMeter.Mark(int64(accountTrieNodesDeleted)) + storageTriesUpdatedMeter.Mark(int64(storageTrieNodesUpdated)) + storageTriesDeletedMeter.Mark(int64(storageTrieNodesDeleted)) + s.AccountUpdated, s.AccountDeleted = 0, 0 + s.StorageUpdated, s.StorageDeleted = 0, 0 - accountUpdatedMeter.Mark(int64(s.AccountUpdated)) - storageUpdatedMeter.Mark(int64(s.StorageUpdated)) - accountDeletedMeter.Mark(int64(s.AccountDeleted)) - storageDeletedMeter.Mark(int64(s.StorageDeleted)) - accountTrieUpdatedMeter.Mark(int64(accountTrieNodesUpdated)) - accountTrieDeletedMeter.Mark(int64(accountTrieNodesDeleted)) - storageTriesUpdatedMeter.Mark(int64(storageTrieNodesUpdated)) - storageTriesDeletedMeter.Mark(int64(storageTrieNodesDeleted)) - s.AccountUpdated, s.AccountDeleted = 0, 0 - s.StorageUpdated, s.StorageDeleted = 0, 0 - } // If snapshotting is enabled, update the snapshot tree with this new version if s.snap != nil { - start := time.Now() + start = time.Now() // Only update if there's a state transition (skip empty Clique blocks) if parent := s.snap.Root(); parent != root { if err := s.snaps.Update(root, parent, s.convertAccountSet(s.stateObjectsDestruct), s.accounts, s.storages); err != nil { @@ -1262,9 +1227,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er log.Warn("Failed to cap snapshot tree", "root", root, "layers", 128, "err", err) } } - if metrics.EnabledExpensive { - s.SnapshotCommits += time.Since(start) - } + s.SnapshotCommits += time.Since(start) s.snap = nil } if root == (common.Hash{}) { @@ -1275,15 +1238,14 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er origin = types.EmptyRootHash } if root != origin { - start := time.Now() - set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete) + start = time.Now() + set := triestate.New(s.accountsOrigin, s.storagesOrigin) if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil { return common.Hash{}, err } s.originalRoot = root - if metrics.EnabledExpensive { - s.TrieDBCommits += time.Since(start) - } + s.TrieDBCommits += time.Since(start) + if s.onCommit != nil { s.onCommit(set) } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index cd86a7f4b67f..3649b0ac589b 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -1161,12 +1161,12 @@ func TestDeleteStorage(t *testing.T) { obj := fastState.getOrNewStateObject(addr) storageRoot := obj.data.Root - _, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } - _, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } diff --git a/core/state/sync.go b/core/state/sync.go index d6775e889610..411b54eab096 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// NewStateSync create a new state trie download scheduler. +// NewStateSync creates a new state trie download scheduler. func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync { // Register the storage slot callback if the external callback is specified. var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error diff --git a/core/state_transition.go b/core/state_transition.go index 2be54480f393..8fcf4c093dbc 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -67,7 +67,7 @@ func (result *ExecutionResult) Revert() []byte { } // IntrinsicGas computes the 'intrinsic gas' for a message with the given data. -func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028 bool, isEIP3860 bool) (uint64, error) { +func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028, isEIP3860 bool) (uint64, error) { // Set the starting gas for the raw transaction var gas uint64 if isContractCreation && isHomestead { @@ -234,7 +234,7 @@ func (st *StateTransition) to() common.Address { func (st *StateTransition) buyGas() error { mgval := new(big.Int).SetUint64(st.msg.GasLimit) - mgval = mgval.Mul(mgval, st.msg.GasPrice) + mgval.Mul(mgval, st.msg.GasPrice) balanceCheck := new(big.Int).Set(mgval) if st.msg.GasFeeCap != nil { balanceCheck.SetUint64(st.msg.GasLimit) @@ -263,7 +263,7 @@ func (st *StateTransition) buyGas() error { if err := st.gp.SubGas(st.msg.GasLimit); err != nil { return err } - st.gasRemaining += st.msg.GasLimit + st.gasRemaining = st.msg.GasLimit st.initialGas = st.msg.GasLimit mgvalU256, _ := uint256.FromBig(mgval) @@ -477,7 +477,7 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { // Return ETH for remaining gas, exchanged at the original rate. remaining := uint256.NewInt(st.gasRemaining) - remaining = remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice)) + remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice)) st.state.AddBalance(st.msg.From, remaining) // Also return remaining gas to the block gas counter so it is diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index ed561f8186da..6dbcc9dadc05 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -360,7 +360,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres } } // Initialize the state with head block, or fallback to empty one in - // case the head state is not available(might occur when node is not + // case the head state is not available (might occur when node is not // fully synced). state, err := p.chain.StateAt(head.Root) if err != nil { @@ -371,7 +371,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres } p.head, p.state = head, state - // Index all transactions on disk and delete anything inprocessable + // Index all transactions on disk and delete anything unprocessable var fails []uint64 index := func(id uint64, size uint32, blob []byte) { if p.parseTransaction(id, size, blob) != nil { @@ -402,7 +402,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres } var ( basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head)) - blobfee = uint256.MustFromBig(big.NewInt(params.BlobTxMinBlobGasprice)) + blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) ) if p.head.ExcessBlobGas != nil { blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(*p.head.ExcessBlobGas)) @@ -540,7 +540,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 } delete(p.index, addr) delete(p.spent, addr) - if inclusions != nil { // only during reorgs will the heap will be initialized + if inclusions != nil { // only during reorgs will the heap be initialized heap.Remove(p.evict, p.evict.index[addr]) } p.reserve(addr, false) @@ -693,7 +693,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 if len(txs) == 0 { delete(p.index, addr) delete(p.spent, addr) - if inclusions != nil { // only during reorgs will the heap will be initialized + if inclusions != nil { // only during reorgs will the heap be initialized heap.Remove(p.evict, p.evict.index[addr]) } p.reserve(addr, false) @@ -809,7 +809,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { } } // Recheck the account's pooled transactions to drop included and - // invalidated one + // invalidated ones p.recheck(addr, inclusions) } if len(adds) > 0 { @@ -1131,8 +1131,12 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error { next = p.state.GetNonce(from) ) if uint64(len(p.index[from])) > tx.Nonce()-next { - // Account can support the replacement, but the price bump must also be met prev := p.index[from][int(tx.Nonce()-next)] + // Ensure the transaction is different than the one tracked locally + if prev.hash == tx.Hash() { + return txpool.ErrAlreadyKnown + } + // Account can support the replacement, but the price bump must also be met switch { case tx.GasFeeCapIntCmp(prev.execFeeCap.ToBig()) <= 0: return fmt.Errorf("%w: new tx gas fee cap %v <= %v queued", txpool.ErrReplaceUnderpriced, tx.GasFeeCap(), prev.execFeeCap) @@ -1226,7 +1230,7 @@ func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error // consensus validity and pool restrictions). func (p *BlobPool) add(tx *types.Transaction) (err error) { // The blob pool blocks on adding a transaction. This is because blob txs are - // only even pulled form the network, so this method will act as the overload + // only even pulled from the network, so this method will act as the overload // protection for fetches. waitStart := time.Now() p.lock.Lock() @@ -1446,7 +1450,12 @@ func (p *BlobPool) drop() { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction { +func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { + // If only plain transactions are requested, this pool is unsuitable as it + // contains none, don't even bother. + if filter.OnlyPlainTxs { + return nil + } // Track the amount of time waiting to retrieve the list of pending blob txs // from the pool and the amount of time actually spent on assembling the data. // The latter will be pretty much moot, but we've kept it to have symmetric @@ -1456,29 +1465,30 @@ func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *u pendwaitHist.Update(time.Since(pendStart).Nanoseconds()) defer p.lock.RUnlock() - defer func(start time.Time) { - pendtimeHist.Update(time.Since(start).Nanoseconds()) - }(time.Now()) + execStart := time.Now() + defer func() { + pendtimeHist.Update(time.Since(execStart).Nanoseconds()) + }() - pending := make(map[common.Address][]*txpool.LazyTransaction) + pending := make(map[common.Address][]*txpool.LazyTransaction, len(p.index)) for addr, txs := range p.index { - var lazies []*txpool.LazyTransaction + lazies := make([]*txpool.LazyTransaction, 0, len(txs)) for _, tx := range txs { // If transaction filtering was requested, discard badly priced ones - if minTip != nil && baseFee != nil { - if tx.execFeeCap.Lt(baseFee) { + if filter.MinTip != nil && filter.BaseFee != nil { + if tx.execFeeCap.Lt(filter.BaseFee) { break // basefee too low, cannot be included, discard rest of txs from the account } - tip := new(uint256.Int).Sub(tx.execFeeCap, baseFee) + tip := new(uint256.Int).Sub(tx.execFeeCap, filter.BaseFee) if tip.Gt(tx.execTipCap) { tip = tx.execTipCap } - if tip.Lt(minTip) { + if tip.Lt(filter.MinTip) { break // allowed or remaining tip too low, cannot be included, discard rest of txs from the account } } - if blobFee != nil { - if tx.blobFeeCap.Lt(blobFee) { + if filter.BlobFee != nil { + if tx.blobFeeCap.Lt(filter.BlobFee) { break // blobfee too low, cannot be included, discard rest of txs from the account } } @@ -1486,9 +1496,9 @@ func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *u lazies = append(lazies, &txpool.LazyTransaction{ Pool: p, Hash: tx.hash, - Time: time.Now(), // TODO(karalabe): Maybe save these and use that? - GasFeeCap: tx.execFeeCap.ToBig(), - GasTipCap: tx.execTipCap.ToBig(), + Time: execStart, // TODO(karalabe): Maybe save these and use that? + GasFeeCap: tx.execFeeCap, + GasTipCap: tx.execTipCap, Gas: tx.execGas, BlobGas: tx.blobGas, }) @@ -1548,7 +1558,7 @@ func (p *BlobPool) updateStorageMetrics() { } // updateLimboMetrics retrieves a bunch of stats from the limbo store and pushes -// // them out as metrics. +// them out as metrics. func (p *BlobPool) updateLimboMetrics() { stats := p.limbo.store.Infos() diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 58353e48289b..279750c73f2a 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -48,7 +48,7 @@ import ( ) var ( - emptyBlob = kzg4844.Blob{} + emptyBlob = new(kzg4844.Blob) emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) emptyBlobVHash = kzg4844.CalcBlobHashV1(sha256.New(), &emptyBlobCommit) @@ -185,7 +185,7 @@ func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64, return types.MustSignNewTx(key, types.LatestSigner(testChainConfig), blobtx) } -// makeUnsignedTx is a utility method to construct a random blob tranasaction +// makeUnsignedTx is a utility method to construct a random blob transaction // without signing it. func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64) *types.BlobTx { return &types.BlobTx{ @@ -198,7 +198,7 @@ func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap BlobHashes: []common.Hash{emptyBlobVHash}, Value: uint256.NewInt(100), Sidecar: &types.BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: []kzg4844.Blob{*emptyBlob}, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, @@ -391,7 +391,7 @@ func TestOpenDrops(t *testing.T) { id, _ := store.Put(blob) filled[id] = struct{}{} } - // Insert a sequence of transactions with partially passed nonces to veirfy + // Insert a sequence of transactions with partially passed nonces to verify // that the included part of the set will get dropped (case 4). var ( overlapper, _ = crypto.GenerateKey() @@ -984,9 +984,14 @@ func TestAdd(t *testing.T) { }, }, adds: []addtx{ - { // New account, 1 tx pending: reject replacement nonce 0 (ignore price for now) + { // New account, 1 tx pending: reject duplicate nonce 0 from: "alice", tx: makeUnsignedTx(0, 1, 1, 1), + err: txpool.ErrAlreadyKnown, + }, + { // New account, 1 tx pending: reject replacement nonce 0 (ignore price for now) + from: "alice", + tx: makeUnsignedTx(0, 1, 1, 2), err: txpool.ErrReplaceUnderpriced, }, { // New account, 1 tx pending: accept nonce 1 @@ -1009,10 +1014,10 @@ func TestAdd(t *testing.T) { tx: makeUnsignedTx(3, 1, 1, 1), err: nil, }, - { // Old account, 1 tx in chain, 1 tx pending: reject replacement nonce 1 (ignore price for now) + { // Old account, 1 tx in chain, 1 tx pending: reject duplicate nonce 1 from: "bob", tx: makeUnsignedTx(1, 1, 1, 1), - err: txpool.ErrReplaceUnderpriced, + err: txpool.ErrAlreadyKnown, }, { // Old account, 1 tx in chain, 1 tx pending: accept nonce 2 (ignore price for now) from: "bob", @@ -1228,6 +1233,24 @@ func TestAdd(t *testing.T) { }, }, }, + // Blob transactions that don't meet the min blob gas price should be rejected + { + seeds: map[string]seed{ + "alice": {balance: 10000000}, + }, + adds: []addtx{ + { // New account, no previous txs, nonce 0, but blob fee cap too low + from: "alice", + tx: makeUnsignedTx(0, 1, 1, 0), + err: txpool.ErrUnderpriced, + }, + { // Same as above but blob fee cap equals minimum, should be accepted + from: "alice", + tx: makeUnsignedTx(0, 1, 1, params.BlobTxMinBlobGasprice), + err: nil, + }, + }, + }, } for i, tt := range tests { // Create a temporary folder for the persistent backend @@ -1288,3 +1311,65 @@ func TestAdd(t *testing.T) { pool.Close() } } + +// Benchmarks the time it takes to assemble the lazy pending transaction list +// from the pool contents. +func BenchmarkPoolPending100Mb(b *testing.B) { benchmarkPoolPending(b, 100_000_000) } +func BenchmarkPoolPending1GB(b *testing.B) { benchmarkPoolPending(b, 1_000_000_000) } +func BenchmarkPoolPending10GB(b *testing.B) { benchmarkPoolPending(b, 10_000_000_000) } + +func benchmarkPoolPending(b *testing.B, datacap uint64) { + // Calculate the maximum number of transaction that would fit into the pool + // and generate a set of random accounts to seed them with. + capacity := datacap / params.BlobTxBlobGasPerBlob + + var ( + basefee = uint64(1050) + blobfee = uint64(105) + signer = types.LatestSigner(testChainConfig) + statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) + chain = &testBlockChain{ + config: testChainConfig, + basefee: uint256.NewInt(basefee), + blobfee: uint256.NewInt(blobfee), + statedb: statedb, + } + pool = New(Config{Datadir: ""}, chain) + ) + + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + b.Fatalf("failed to create blob pool: %v", err) + } + // Fill the pool up with one random transaction from each account with the + // same price and everything to maximize the worst case scenario + for i := 0; i < int(capacity); i++ { + blobtx := makeUnsignedTx(0, 10, basefee+10, blobfee) + blobtx.R = uint256.NewInt(1) + blobtx.S = uint256.NewInt(uint64(100 + i)) + blobtx.V = uint256.NewInt(0) + tx := types.NewTx(blobtx) + addr, err := types.Sender(signer, tx) + if err != nil { + b.Fatal(err) + } + statedb.AddBalance(addr, uint256.NewInt(1_000_000_000)) + pool.add(tx) + } + statedb.Commit(0, true) + defer pool.Close() + + // Benchmark assembling the pending + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + p := pool.Pending(txpool.PendingFilter{ + MinTip: uint256.NewInt(1), + BaseFee: chain.basefee, + BlobFee: chain.blobfee, + }) + if len(p) != int(capacity) { + b.Fatalf("have %d want %d", len(p), capacity) + } + } +} diff --git a/core/txpool/blobpool/config.go b/core/txpool/blobpool/config.go index 99a2002a303f..1d180739cdfb 100644 --- a/core/txpool/blobpool/config.go +++ b/core/txpool/blobpool/config.go @@ -30,8 +30,8 @@ type Config struct { // DefaultConfig contains the default configurations for the transaction pool. var DefaultConfig = Config{ Datadir: "blobpool", - Datacap: 10 * 1024 * 1024 * 1024, - PriceBump: 100, // either have patience or be aggressive, no mushy ground + Datacap: 10 * 1024 * 1024 * 1024 / 4, // TODO(karalabe): /4 handicap for rollout, gradually bump back up to 10GB + PriceBump: 100, // either have patience or be aggressive, no mushy ground } // sanitize checks the provided user configurations and changes anything that's diff --git a/core/txpool/blobpool/evictheap.go b/core/txpool/blobpool/evictheap.go index df594099f79b..bc4543a352e2 100644 --- a/core/txpool/blobpool/evictheap.go +++ b/core/txpool/blobpool/evictheap.go @@ -30,7 +30,7 @@ import ( // transaction from each account to determine which account to evict from. // // The heap internally tracks a slice of cheapest transactions from each account -// and a mapping from addresses to indices for direct removals/udates. +// and a mapping from addresses to indices for direct removals/updates. // // The goal of the heap is to decide which account has the worst bottleneck to // evict transactions from. diff --git a/core/txpool/blobpool/priority_test.go b/core/txpool/blobpool/priority_test.go index 4aad919925f5..cf0e0454a00a 100644 --- a/core/txpool/blobpool/priority_test.go +++ b/core/txpool/blobpool/priority_test.go @@ -64,7 +64,7 @@ func BenchmarkDynamicFeeJumpCalculation(b *testing.B) { // Benchmarks how many priority recalculations can be done. func BenchmarkPriorityCalculation(b *testing.B) { // The basefee and blob fee is constant for all transactions across a block, - // so we can assume theit absolute jump counts can be pre-computed. + // so we can assume their absolute jump counts can be pre-computed. basefee := uint256.NewInt(17_200_000_000) // 17.2 Gwei is the 22.03.2023 zero-emission basefee, random number blobfee := uint256.NewInt(123_456_789_000) // Completely random, no idea what this will be diff --git a/core/txpool/errors.go b/core/txpool/errors.go index 61daa999ffdc..3a6a913976ca 100644 --- a/core/txpool/errors.go +++ b/core/txpool/errors.go @@ -54,4 +54,10 @@ var ( // ErrFutureReplacePending is returned if a future transaction replaces a pending // one. Future transactions should only be able to replace other future transactions. ErrFutureReplacePending = errors.New("future transaction tries to replace pending") + + // ErrAlreadyReserved is returned if the sender address has a pending transaction + // in a different subpool. For example, this error is returned in response to any + // input transaction of non-blob type when a blob transaction from this sender + // remains pending (and vice-versa). + ErrAlreadyReserved = errors.New("address already reserved") ) diff --git a/core/txpool/legacypool/journal.go b/core/txpool/legacypool/journal.go index f04ab8fc14e5..899ed00bcced 100644 --- a/core/txpool/legacypool/journal.go +++ b/core/txpool/legacypool/journal.go @@ -164,7 +164,12 @@ func (journal *journal) rotate(all map[common.Address]types.Transactions) error return err } journal.writer = sink - log.Info("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all)) + + logger := log.Info + if len(all) == 0 { + logger = log.Debug + } + logger("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all)) return nil } diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 18ca27a11a14..4e1d26acf405 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -296,7 +296,7 @@ func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserve txpool.A pool.gasTip.Store(uint256.NewInt(gasTip)) // Initialize the state with head block, or fallback to empty one in - // case the head state is not available(might occur when node is not + // case the head state is not available (might occur when node is not // fully synced). statedb, err := pool.chain.StateAt(head.Root) if err != nil { @@ -522,7 +522,12 @@ func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction, // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction { +func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { + // If only blob transactions are requested, this pool is unsuitable as it + // contains none, don't even bother. + if filter.OnlyBlobTxs { + return nil + } pool.mu.Lock() defer pool.mu.Unlock() @@ -531,13 +536,12 @@ func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobF minTipBig *big.Int baseFeeBig *big.Int ) - if minTip != nil { - minTipBig = minTip.ToBig() + if filter.MinTip != nil { + minTipBig = filter.MinTip.ToBig() } - if baseFee != nil { - baseFeeBig = baseFee.ToBig() + if filter.BaseFee != nil { + baseFeeBig = filter.BaseFee.ToBig() } - pending := make(map[common.Address][]*txpool.LazyTransaction, len(pool.pending)) for addr, list := range pool.pending { txs := list.Flatten() @@ -559,8 +563,8 @@ func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobF Hash: txs[i].Hash(), Tx: txs[i], Time: txs[i].Time(), - GasFeeCap: txs[i].GasFeeCap(), - GasTipCap: txs[i].GasTipCap(), + GasFeeCap: uint256.MustFromBig(txs[i].GasFeeCap()), + GasTipCap: uint256.MustFromBig(txs[i].GasTipCap()), Gas: txs[i].Gas(), BlobGas: txs[i].BlobGas(), } diff --git a/core/txpool/legacypool/list.go b/core/txpool/legacypool/list.go index f0f9f213f27d..7db9c98ace63 100644 --- a/core/txpool/legacypool/list.go +++ b/core/txpool/legacypool/list.go @@ -278,7 +278,7 @@ type list struct { totalcost *uint256.Int // Total cost of all transactions in the list } -// newList create a new transaction list for maintaining nonce-indexable fast, +// newList creates a new transaction list for maintaining nonce-indexable fast, // gapped, sortable transaction lists. func newList(strict bool) *list { return &list{ diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index aa19eef0d06a..9881ed1b8f96 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -35,9 +35,9 @@ type LazyTransaction struct { Hash common.Hash // Transaction hash to pull up if needed Tx *types.Transaction // Transaction if already resolved - Time time.Time // Time when the transaction was first seen - GasFeeCap *big.Int // Maximum fee per gas the transaction may consume - GasTipCap *big.Int // Maximum miner tip per gas the transaction can pay + Time time.Time // Time when the transaction was first seen + GasFeeCap *uint256.Int // Maximum fee per gas the transaction may consume + GasTipCap *uint256.Int // Maximum miner tip per gas the transaction can pay Gas uint64 // Amount of gas required by the transaction BlobGas uint64 // Amount of blob gas required by the transaction @@ -70,6 +70,21 @@ type LazyResolver interface { // may request (and relinquish) exclusive access to certain addresses. type AddressReserver func(addr common.Address, reserve bool) error +// PendingFilter is a collection of filter rules to allow retrieving a subset +// of transactions for announcement or mining. +// +// Note, the entries here are not arbitrary useful filters, rather each one has +// a very specific call site in mind and each one can be evaluated very cheaply +// by the pool implementations. Only add new ones that satisfy those constraints. +type PendingFilter struct { + MinTip *uint256.Int // Minimum miner tip required to include a transaction + BaseFee *uint256.Int // Minimum 1559 basefee needed to include a transaction + BlobFee *uint256.Int // Minimum 4844 blobfee needed to include a blob transaction + + OnlyPlainTxs bool // Return only plain EVM transactions (peer-join announces, block space filling) + OnlyBlobTxs bool // Return only blob transactions (block blob-space filling) +} + // SubPool represents a specialized transaction pool that lives on its own (e.g. // blob pool). Since independent of how many specialized pools we have, they do // need to be updated in lockstep and assemble into one coherent view for block @@ -118,7 +133,7 @@ type SubPool interface { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. - Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*LazyTransaction + Pending(filter PendingFilter) map[common.Address][]*LazyTransaction // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 3d0d6bf617bd..be7435247d92 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "github.com/holiman/uint256" ) // TxStatus is the current status of a transaction as seen by the pool. @@ -123,7 +122,7 @@ func (p *TxPool) reserver(id int, subpool SubPool) AddressReserver { log.Error("pool attempted to reserve already-owned address", "address", addr) return nil // Ignore fault to give the pool a chance to recover while the bug gets fixed } - return errors.New("address already reserved") + return ErrAlreadyReserved } p.reservations[addr] = subpool if metrics.Enabled { @@ -357,10 +356,10 @@ func (p *TxPool) Add(txs []*types.Transaction, local bool, sync bool) []error { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (p *TxPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*LazyTransaction { +func (p *TxPool) Pending(filter PendingFilter) map[common.Address][]*LazyTransaction { txs := make(map[common.Address][]*LazyTransaction) for _, subpool := range p.subpools { - for addr, set := range subpool.Pending(minTip, baseFee, blobFee) { + for addr, set := range subpool.Pending(filter) { txs[addr] = set } } diff --git a/core/txpool/validation.go b/core/txpool/validation.go index a9bd14020bc9..d9a85a435da4 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -18,6 +18,7 @@ package txpool import ( "crypto/sha256" + "errors" "fmt" "math/big" @@ -30,6 +31,12 @@ import ( "github.com/ethereum/go-ethereum/params" ) +var ( + // blobTxMinBlobGasPrice is the big.Int version of the configured protocol + // parameter to avoid constucting a new big integer for every transaction. + blobTxMinBlobGasPrice = big.NewInt(params.BlobTxMinBlobGasprice) +) + // ValidationOptions define certain differences between transaction validation // across the different pools without having to duplicate those checks. type ValidationOptions struct { @@ -101,28 +108,31 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return err } if tx.Gas() < intrGas { - return fmt.Errorf("%w: needed %v, allowed %v", core.ErrIntrinsicGas, intrGas, tx.Gas()) + return fmt.Errorf("%w: gas %v, minimum needed %v", core.ErrIntrinsicGas, tx.Gas(), intrGas) } - // Ensure the gasprice is high enough to cover the requirement of the calling - // pool and/or block producer + // Ensure the gasprice is high enough to cover the requirement of the calling pool if tx.GasTipCapIntCmp(opts.MinTip) < 0 { - return fmt.Errorf("%w: tip needed %v, tip permitted %v", ErrUnderpriced, opts.MinTip, tx.GasTipCap()) + return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrUnderpriced, tx.GasTipCap(), opts.MinTip) } - // Ensure blob transactions have valid commitments if tx.Type() == types.BlobTxType { + // Ensure the blob fee cap satisfies the minimum blob gas price + if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 { + return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrUnderpriced, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) + } sidecar := tx.BlobTxSidecar() if sidecar == nil { - return fmt.Errorf("missing sidecar in blob transaction") + return errors.New("missing sidecar in blob transaction") } // Ensure the number of items in the blob transaction and various side // data match up before doing any expensive validations hashes := tx.BlobHashes() if len(hashes) == 0 { - return fmt.Errorf("blobless blob transaction") + return errors.New("blobless blob transaction") } if len(hashes) > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob { return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) } + // Ensure commitments, proofs and hashes are valid if err := validateBlobSidecar(hashes, sidecar); err != nil { return err } @@ -152,7 +162,7 @@ func validateBlobSidecar(hashes []common.Hash, sidecar *types.BlobTxSidecar) err // Blob commitments match with the hashes in the transaction, verify the // blobs themselves via KZG for i := range sidecar.Blobs { - if err := kzg4844.VerifyBlobProof(sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil { + if err := kzg4844.VerifyBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil { return fmt.Errorf("invalid blob %d: %v", i, err) } } diff --git a/core/types/account.go b/core/types/account.go index bb0f4ca02e10..52ce184cda5d 100644 --- a/core/types/account.go +++ b/core/types/account.go @@ -52,7 +52,7 @@ type accountMarshaling struct { } // storageJSON represents a 256 bit byte array, but allows less than 256 bits when -// unmarshaling from hex. +// unmarshalling from hex. type storageJSON common.Hash func (h *storageJSON) UnmarshalText(text []byte) error { diff --git a/core/types/block_test.go b/core/types/block_test.go index cf0b1dd85c1e..982d002242f6 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -196,7 +196,7 @@ func TestEIP2718BlockEncoding(t *testing.T) { func TestUncleHash(t *testing.T) { uncles := make([]*Header, 0) h := CalcUncleHash(uncles) - exp := common.HexToHash("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347") + exp := EmptyUncleHash if h != exp { t.Fatalf("empty uncle hash is wrong, got %x != %x", h, exp) } diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index a7b26444712f..fc51eb11a528 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -343,7 +343,7 @@ func TestReceiptJSON(t *testing.T) { r := Receipt{} err = r.UnmarshalJSON(b) if err != nil { - t.Fatal("error unmarshaling receipt from json:", err) + t.Fatal("error unmarshalling receipt from json:", err) } } } @@ -360,7 +360,7 @@ func TestEffectiveGasPriceNotRequired(t *testing.T) { r2 := Receipt{} err = r2.UnmarshalJSON(b) if err != nil { - t.Fatal("error unmarshaling receipt from json:", err) + t.Fatal("error unmarshalling receipt from json:", err) } } diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 9e26642f753d..70dee0776e7b 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -107,13 +107,7 @@ func SignTx(tx *Transaction, s Signer, prv *ecdsa.PrivateKey) (*Transaction, err // SignNewTx creates a transaction and signs it. func SignNewTx(prv *ecdsa.PrivateKey, s Signer, txdata TxData) (*Transaction, error) { - tx := NewTx(txdata) - h := s.Hash(tx) - sig, err := crypto.Sign(h[:], prv) - if err != nil { - return nil, err - } - return tx.WithSignature(s, sig) + return SignTx(NewTx(txdata), s, prv) } // MustSignNewTx creates a transaction and signs it. diff --git a/core/types/tx_blob_test.go b/core/types/tx_blob_test.go index 25d09e31ce4a..6bd0f183b730 100644 --- a/core/types/tx_blob_test.go +++ b/core/types/tx_blob_test.go @@ -59,7 +59,7 @@ func TestBlobTxSize(t *testing.T) { } var ( - emptyBlob = kzg4844.Blob{} + emptyBlob = new(kzg4844.Blob) emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) ) @@ -72,7 +72,7 @@ func createEmptyBlobTx(key *ecdsa.PrivateKey, withSidecar bool) *Transaction { func createEmptyBlobTxInner(withSidecar bool) *BlobTx { sidecar := &BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: []kzg4844.Blob{*emptyBlob}, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, } diff --git a/core/vm/errors.go b/core/vm/errors.go index fbbf19e178bf..004f8ef1c83c 100644 --- a/core/vm/errors.go +++ b/core/vm/errors.go @@ -29,7 +29,6 @@ var ( ErrInsufficientBalance = errors.New("insufficient balance for transfer") ErrContractAddressCollision = errors.New("contract address collision") ErrExecutionReverted = errors.New("execution reverted") - ErrMaxInitCodeSizeExceeded = errors.New("max initcode size exceeded") ErrMaxCodeSizeExceeded = errors.New("max code size exceeded") ErrInvalidJump = errors.New("invalid jump destination") ErrWriteProtection = errors.New("write protection") diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 023aa0af0008..ac3ea4bcd62b 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -17,6 +17,8 @@ package vm import ( + "math" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -303,7 +305,7 @@ func opCallDataCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext ) dataOffset64, overflow := dataOffset.Uint64WithOverflow() if overflow { - dataOffset64 = 0xffffffffffffffff + dataOffset64 = math.MaxUint64 } // These values are checked for overflow during gas cost calculation memOffset64 := memOffset.Uint64() @@ -359,7 +361,7 @@ func opCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ ) uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() if overflow { - uint64CodeOffset = 0xffffffffffffffff + uint64CodeOffset = math.MaxUint64 } codeCopy := getData(scope.Contract.Code, uint64CodeOffset, length.Uint64()) scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) @@ -377,7 +379,7 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ) uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() if overflow { - uint64CodeOffset = 0xffffffffffffffff + uint64CodeOffset = math.MaxUint64 } addr := common.Address(a.Bytes20()) codeCopy := getData(interpreter.evm.StateDB.GetCode(addr), uint64CodeOffset, length.Uint64()) diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index bca6d1e83b88..f420a241058b 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -187,7 +187,12 @@ func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc { // outside of this function, as part of the dynamic gas, and that will make it // also become correctly reported to tracers. contract.Gas += coldCost - return gas + coldCost, nil + + var overflow bool + if gas, overflow = math.SafeAdd(gas, coldCost); overflow { + return 0, ErrGasUintOverflow + } + return gas, nil } } diff --git a/crypto/bn256/cloudflare/gfp_decl.go b/crypto/bn256/cloudflare/gfp_decl.go index cf7f5654239f..1954d14a4a5a 100644 --- a/crypto/bn256/cloudflare/gfp_decl.go +++ b/crypto/bn256/cloudflare/gfp_decl.go @@ -13,7 +13,7 @@ import ( //nolint:varcheck,unused,deadcode var hasBMI2 = cpu.X86.HasBMI2 -// go:noescape +//go:noescape func gfpNeg(c, a *gfP) //go:noescape diff --git a/crypto/kzg4844/kzg4844.go b/crypto/kzg4844/kzg4844.go index 52124df67461..39fdfbe740ee 100644 --- a/crypto/kzg4844/kzg4844.go +++ b/crypto/kzg4844/kzg4844.go @@ -85,7 +85,7 @@ type Claim [32]byte var useCKZG atomic.Bool // UseCKZG can be called to switch the default Go implementation of KZG to the C -// library if fo some reason the user wishes to do so (e.g. consensus bug in one +// library if for some reason the user wishes to do so (e.g. consensus bug in one // or the other). func UseCKZG(use bool) error { if use && !ckzgAvailable { @@ -105,7 +105,7 @@ func UseCKZG(use bool) error { } // BlobToCommitment creates a small commitment out of a data blob. -func BlobToCommitment(blob Blob) (Commitment, error) { +func BlobToCommitment(blob *Blob) (Commitment, error) { if useCKZG.Load() { return ckzgBlobToCommitment(blob) } @@ -114,7 +114,7 @@ func BlobToCommitment(blob Blob) (Commitment, error) { // ComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func ComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func ComputeProof(blob *Blob, point Point) (Proof, Claim, error) { if useCKZG.Load() { return ckzgComputeProof(blob, point) } @@ -134,7 +134,7 @@ func VerifyProof(commitment Commitment, point Point, claim Claim, proof Proof) e // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func ComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func ComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { if useCKZG.Load() { return ckzgComputeBlobProof(blob, commitment) } @@ -142,7 +142,7 @@ func ComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { } // VerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func VerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func VerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { if useCKZG.Load() { return ckzgVerifyBlobProof(blob, commitment, proof) } diff --git a/crypto/kzg4844/kzg4844_ckzg_cgo.go b/crypto/kzg4844/kzg4844_ckzg_cgo.go index 54002856987c..11bc451b58a9 100644 --- a/crypto/kzg4844/kzg4844_ckzg_cgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_cgo.go @@ -61,10 +61,10 @@ func ckzgInit() { } // ckzgBlobToCommitment creates a small commitment out of a data blob. -func ckzgBlobToCommitment(blob Blob) (Commitment, error) { +func ckzgBlobToCommitment(blob *Blob) (Commitment, error) { ckzgIniter.Do(ckzgInit) - commitment, err := ckzg4844.BlobToKZGCommitment((ckzg4844.Blob)(blob)) + commitment, err := ckzg4844.BlobToKZGCommitment((*ckzg4844.Blob)(blob)) if err != nil { return Commitment{}, err } @@ -73,10 +73,10 @@ func ckzgBlobToCommitment(blob Blob) (Commitment, error) { // ckzgComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func ckzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func ckzgComputeProof(blob *Blob, point Point) (Proof, Claim, error) { ckzgIniter.Do(ckzgInit) - proof, claim, err := ckzg4844.ComputeKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes32)(point)) + proof, claim, err := ckzg4844.ComputeKZGProof((*ckzg4844.Blob)(blob), (ckzg4844.Bytes32)(point)) if err != nil { return Proof{}, Claim{}, err } @@ -102,10 +102,10 @@ func ckzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Proo // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func ckzgComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { ckzgIniter.Do(ckzgInit) - proof, err := ckzg4844.ComputeBlobKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment)) + proof, err := ckzg4844.ComputeBlobKZGProof((*ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment)) if err != nil { return Proof{}, err } @@ -113,10 +113,10 @@ func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { } // ckzgVerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func ckzgVerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func ckzgVerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { ckzgIniter.Do(ckzgInit) - valid, err := ckzg4844.VerifyBlobKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment), (ckzg4844.Bytes48)(proof)) + valid, err := ckzg4844.VerifyBlobKZGProof((*ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment), (ckzg4844.Bytes48)(proof)) if err != nil { return err } diff --git a/crypto/kzg4844/kzg4844_ckzg_nocgo.go b/crypto/kzg4844/kzg4844_ckzg_nocgo.go index ed840c75bb68..70a78e80d16a 100644 --- a/crypto/kzg4844/kzg4844_ckzg_nocgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_nocgo.go @@ -32,13 +32,13 @@ func ckzgInit() { } // ckzgBlobToCommitment creates a small commitment out of a data blob. -func ckzgBlobToCommitment(blob Blob) (Commitment, error) { +func ckzgBlobToCommitment(blob *Blob) (Commitment, error) { panic("unsupported platform") } // ckzgComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func ckzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func ckzgComputeProof(blob *Blob, point Point) (Proof, Claim, error) { panic("unsupported platform") } @@ -52,11 +52,11 @@ func ckzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Proo // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func ckzgComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { panic("unsupported platform") } // ckzgVerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func ckzgVerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func ckzgVerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { panic("unsupported platform") } diff --git a/crypto/kzg4844/kzg4844_gokzg.go b/crypto/kzg4844/kzg4844_gokzg.go index 3f03bb52738e..b4af9b1671e9 100644 --- a/crypto/kzg4844/kzg4844_gokzg.go +++ b/crypto/kzg4844/kzg4844_gokzg.go @@ -46,10 +46,10 @@ func gokzgInit() { } // gokzgBlobToCommitment creates a small commitment out of a data blob. -func gokzgBlobToCommitment(blob Blob) (Commitment, error) { +func gokzgBlobToCommitment(blob *Blob) (Commitment, error) { gokzgIniter.Do(gokzgInit) - commitment, err := context.BlobToKZGCommitment((gokzg4844.Blob)(blob), 0) + commitment, err := context.BlobToKZGCommitment((*gokzg4844.Blob)(blob), 0) if err != nil { return Commitment{}, err } @@ -58,10 +58,10 @@ func gokzgBlobToCommitment(blob Blob) (Commitment, error) { // gokzgComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. -func gokzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { +func gokzgComputeProof(blob *Blob, point Point) (Proof, Claim, error) { gokzgIniter.Do(gokzgInit) - proof, claim, err := context.ComputeKZGProof((gokzg4844.Blob)(blob), (gokzg4844.Scalar)(point), 0) + proof, claim, err := context.ComputeKZGProof((*gokzg4844.Blob)(blob), (gokzg4844.Scalar)(point), 0) if err != nil { return Proof{}, Claim{}, err } @@ -80,10 +80,10 @@ func gokzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Pro // the commitment. // // This method does not verify that the commitment is correct with respect to blob. -func gokzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { +func gokzgComputeBlobProof(blob *Blob, commitment Commitment) (Proof, error) { gokzgIniter.Do(gokzgInit) - proof, err := context.ComputeBlobKZGProof((gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), 0) + proof, err := context.ComputeBlobKZGProof((*gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), 0) if err != nil { return Proof{}, err } @@ -91,8 +91,8 @@ func gokzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { } // gokzgVerifyBlobProof verifies that the blob data corresponds to the provided commitment. -func gokzgVerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { +func gokzgVerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { gokzgIniter.Do(gokzgInit) - return context.VerifyBlobKZGProof((gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), (gokzg4844.KZGProof)(proof)) + return context.VerifyBlobKZGProof((*gokzg4844.Blob)(blob), (gokzg4844.KZGCommitment)(commitment), (gokzg4844.KZGProof)(proof)) } diff --git a/crypto/kzg4844/kzg4844_test.go b/crypto/kzg4844/kzg4844_test.go index fae8a7a76eaf..a6782d4768ad 100644 --- a/crypto/kzg4844/kzg4844_test.go +++ b/crypto/kzg4844/kzg4844_test.go @@ -36,13 +36,13 @@ func randFieldElement() [32]byte { return gokzg4844.SerializeScalar(r) } -func randBlob() Blob { +func randBlob() *Blob { var blob Blob for i := 0; i < len(blob); i += gokzg4844.SerializedScalarSize { fieldElementBytes := randFieldElement() copy(blob[i:i+gokzg4844.SerializedScalarSize], fieldElementBytes[:]) } - return blob + return &blob } func TestCKZGWithPoint(t *testing.T) { testKZGWithPoint(t, true) } diff --git a/crypto/secp256k1/libsecp256k1/include/secp256k1.h b/crypto/secp256k1/libsecp256k1/include/secp256k1.h index f268e309d0bf..76af8396918e 100644 --- a/crypto/secp256k1/libsecp256k1/include/secp256k1.h +++ b/crypto/secp256k1/libsecp256k1/include/secp256k1.h @@ -357,7 +357,7 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact( /** Verify an ECDSA signature. * * Returns: 1: correct signature - * 0: incorrect or unparseable signature + * 0: incorrect or unparsable signature * Args: ctx: a secp256k1 context object, initialized for verification. * In: sig: the signature being verified (cannot be NULL) * msg32: the 32-byte message hash being verified (cannot be NULL) diff --git a/crypto/secp256k1/libsecp256k1/sage/group_prover.sage b/crypto/secp256k1/libsecp256k1/sage/group_prover.sage index ab580c5b23bb..68882e93659a 100644 --- a/crypto/secp256k1/libsecp256k1/sage/group_prover.sage +++ b/crypto/secp256k1/libsecp256k1/sage/group_prover.sage @@ -17,7 +17,7 @@ # - A constraint describing the requirements of the law, called "require" # * Implementations are transliterated into functions that operate as well on # algebraic input points, and are called once per combination of branches -# exectured. Each execution returns: +# executed. Each execution returns: # - A constraint describing the assumptions this implementation requires # (such as Z1=1), called "assumeFormula" # - A constraint describing the assumptions this specific branch requires, diff --git a/eth/api.go b/eth/api.go deleted file mode 100644 index 44e934fd040b..000000000000 --- a/eth/api.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" -) - -// EthereumAPI provides an API to access Ethereum full node-related information. -type EthereumAPI struct { - e *Ethereum -} - -// NewEthereumAPI creates a new Ethereum protocol API for full nodes. -func NewEthereumAPI(e *Ethereum) *EthereumAPI { - return &EthereumAPI{e} -} - -// Etherbase is the address that mining rewards will be sent to. -func (api *EthereumAPI) Etherbase() (common.Address, error) { - return api.e.Etherbase() -} - -// Coinbase is the address that mining rewards will be sent to (alias for Etherbase). -func (api *EthereumAPI) Coinbase() (common.Address, error) { - return api.Etherbase() -} - -// Hashrate returns the POW hashrate. -func (api *EthereumAPI) Hashrate() hexutil.Uint64 { - return hexutil.Uint64(api.e.Miner().Hashrate()) -} - -// Mining returns an indication if this node is currently mining. -func (api *EthereumAPI) Mining() bool { - return api.e.IsMining() -} diff --git a/eth/api_backend.go b/eth/api_backend.go index c24fa313936f..48c46447c5a0 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -37,7 +37,6 @@ import ( "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -67,7 +66,7 @@ func (b *EthAPIBackend) SetHead(number uint64) { func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { // Pending block is only known by the miner if number == rpc.PendingBlockNumber { - block := b.eth.miner.PendingBlock() + block, _, _ := b.eth.miner.Pending() if block == nil { return nil, errors.New("pending block is not available") } @@ -118,7 +117,7 @@ func (b *EthAPIBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*ty func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { // Pending block is only known by the miner if number == rpc.PendingBlockNumber { - block := b.eth.miner.PendingBlock() + block, _, _ := b.eth.miner.Pending() if block == nil { return nil, errors.New("pending block is not available") } @@ -182,14 +181,14 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r return nil, errors.New("invalid arguments; neither block nor hash specified") } -func (b *EthAPIBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { - return b.eth.miner.PendingBlockAndReceipts() +func (b *EthAPIBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { + return b.eth.miner.Pending() } func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { // Pending state is only known by the miner if number == rpc.PendingBlockNumber { - block, state := b.eth.miner.Pending() + block, _, state := b.eth.miner.Pending() if block == nil || state == nil { return nil, nil, errors.New("pending state is not available") } @@ -267,10 +266,6 @@ func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEven return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch) } -func (b *EthAPIBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - return b.eth.miner.SubscribePendingLogs(ch) -} - func (b *EthAPIBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainEvent(ch) } @@ -292,7 +287,7 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { - pending := b.eth.txPool.Pending(nil, nil, nil) + pending := b.eth.txPool.Pending(txpool.PendingFilter{}) var txs types.Transactions for _, batch := range pending { for _, lazy := range batch { @@ -421,14 +416,6 @@ func (b *EthAPIBackend) CurrentHeader() *types.Header { return b.eth.blockchain.CurrentHeader() } -func (b *EthAPIBackend) Miner() *miner.Miner { - return b.eth.Miner() -} - -func (b *EthAPIBackend) StartMining() error { - return b.eth.StartMining() -} - func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk) } diff --git a/eth/api_debug.go b/eth/api_debug.go index 05010a3969c6..d5e4dda1401c 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -56,7 +56,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { // If we're dumping the pending state, we need to request // both the pending block as well as the pending state from // the miner and operate on those - _, stateDb := api.eth.miner.Pending() + _, _, stateDb := api.eth.miner.Pending() if stateDb == nil { return state.Dump{}, errors.New("pending state is not available") } @@ -142,7 +142,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex // If we're dumping the pending state, we need to request // both the pending block as well as the pending state from // the miner and operate on those - _, stateDb = api.eth.miner.Pending() + _, _, stateDb = api.eth.miner.Pending() if stateDb == nil { return state.Dump{}, errors.New("pending state is not available") } diff --git a/eth/api_miner.go b/eth/api_miner.go index 2fe296548a20..8c96f4c54aff 100644 --- a/eth/api_miner.go +++ b/eth/api_miner.go @@ -18,9 +18,7 @@ package eth import ( "math/big" - "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" ) @@ -29,26 +27,11 @@ type MinerAPI struct { e *Ethereum } -// NewMinerAPI create a new MinerAPI instance. +// NewMinerAPI creates a new MinerAPI instance. func NewMinerAPI(e *Ethereum) *MinerAPI { return &MinerAPI{e} } -// Start starts the miner with the given number of threads. If threads is nil, -// the number of workers started is equal to the number of logical CPUs that are -// usable by this process. If mining is already running, this method adjust the -// number of threads allowed to use and updates the minimum price required by the -// transaction pool. -func (api *MinerAPI) Start() error { - return api.e.StartMining() -} - -// Stop terminates the miner, both at the consensus engine level as well as at -// the block creation level. -func (api *MinerAPI) Stop() { - api.e.StopMining() -} - // SetExtra sets the extra data string that is included when this miner mines a block. func (api *MinerAPI) SetExtra(extra string) (bool, error) { if err := api.e.Miner().SetExtra([]byte(extra)); err != nil { @@ -73,14 +56,3 @@ func (api *MinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool { api.e.Miner().SetGasCeil(uint64(gasLimit)) return true } - -// SetEtherbase sets the etherbase of the miner. -func (api *MinerAPI) SetEtherbase(etherbase common.Address) bool { - api.e.SetEtherbase(etherbase) - return true -} - -// SetRecommitInterval updates the interval for miner sealing work recommitting. -func (api *MinerAPI) SetRecommitInterval(interval int) { - api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond) -} diff --git a/eth/backend.go b/eth/backend.go index 565d81ca07a2..f8a7ad96473f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -28,8 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/beacon" - "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" @@ -74,7 +72,6 @@ type Ethereum struct { handler *handler ethDialCandidates enode.Iterator snapDialCandidates enode.Iterator - merger *consensus.Merger // DB interfaces chainDb ethdb.Database // Block chain database @@ -89,9 +86,8 @@ type Ethereum struct { APIBackend *EthAPIBackend - miner *miner.Miner - gasPrice *big.Int - etherbase common.Address + miner *miner.Miner + gasPrice *big.Int networkID uint64 netRPCService *ethapi.NetAPI @@ -158,7 +154,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } eth := &Ethereum{ config: config, - merger: consensus.NewMerger(chainDb), chainDb: chainDb, eventMux: stack.EventMux(), accountManager: stack.AccountManager(), @@ -166,7 +161,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { closeBloomHandler: make(chan struct{}), networkID: networkID, gasPrice: config.Miner.GasPrice, - etherbase: config.Miner.Etherbase, bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms), p2pServer: stack.Server(), @@ -213,7 +207,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if config.OverrideVerkle != nil { overrides.OverrideVerkle = config.OverrideVerkle } - eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TransactionHistory) + // TODO (MariusVanDerWijden) get rid of shouldPreserve in a follow-up PR + shouldPreserve := func(header *types.Header) bool { + return false + } + eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, shouldPreserve, &config.TransactionHistory) if err != nil { return nil, err } @@ -236,10 +234,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // Permit the downloader to use the trie cache allowance during fast sync cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit if eth.handler, err = newHandler(&handlerConfig{ + NodeID: eth.p2pServer.Self().ID(), Database: chainDb, Chain: eth.blockchain, TxPool: eth.txPool, - Merger: eth.merger, Network: networkID, Sync: config.SyncMode, BloomCache: uint64(cacheLimit), @@ -249,7 +247,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, &config.Miner, eth.blockchain.Config(), eth.EventMux(), eth.engine, eth.isLocalBlock) + eth.miner = miner.New(eth, config.Miner, eth.engine) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil} @@ -315,9 +313,6 @@ func (s *Ethereum) APIs() []rpc.API { // Append all the local APIs and return return append(apis, []rpc.API{ { - Namespace: "eth", - Service: NewEthereumAPI(s), - }, { Namespace: "miner", Service: NewMinerAPI(s), }, { @@ -344,138 +339,6 @@ func (s *Ethereum) ResetWithGenesisBlock(gb *types.Block) { s.blockchain.ResetWithGenesisBlock(gb) } -func (s *Ethereum) Etherbase() (eb common.Address, err error) { - s.lock.RLock() - etherbase := s.etherbase - s.lock.RUnlock() - - if etherbase != (common.Address{}) { - return etherbase, nil - } - return common.Address{}, errors.New("etherbase must be explicitly specified") -} - -// isLocalBlock checks whether the specified block is mined -// by local miner accounts. -// -// We regard two types of accounts as local miner account: etherbase -// and accounts specified via `txpool.locals` flag. -func (s *Ethereum) isLocalBlock(header *types.Header) bool { - author, err := s.engine.Author(header) - if err != nil { - log.Warn("Failed to retrieve block author", "number", header.Number.Uint64(), "hash", header.Hash(), "err", err) - return false - } - // Check whether the given address is etherbase. - s.lock.RLock() - etherbase := s.etherbase - s.lock.RUnlock() - if author == etherbase { - return true - } - // Check whether the given address is specified by `txpool.local` - // CLI flag. - for _, account := range s.config.TxPool.Locals { - if account == author { - return true - } - } - return false -} - -// shouldPreserve checks whether we should preserve the given block -// during the chain reorg depending on whether the author of block -// is a local account. -func (s *Ethereum) shouldPreserve(header *types.Header) bool { - // The reason we need to disable the self-reorg preserving for clique - // is it can be probable to introduce a deadlock. - // - // e.g. If there are 7 available signers - // - // r1 A - // r2 B - // r3 C - // r4 D - // r5 A [X] F G - // r6 [X] - // - // In the round5, the in-turn signer E is offline, so the worst case - // is A, F and G sign the block of round5 and reject the block of opponents - // and in the round6, the last available signer B is offline, the whole - // network is stuck. - if _, ok := s.engine.(*clique.Clique); ok { - return false - } - return s.isLocalBlock(header) -} - -// SetEtherbase sets the mining reward address. -func (s *Ethereum) SetEtherbase(etherbase common.Address) { - s.lock.Lock() - s.etherbase = etherbase - s.lock.Unlock() - - s.miner.SetEtherbase(etherbase) -} - -// StartMining starts the miner with the given number of CPU threads. If mining -// is already running, this method adjust the number of threads allowed to use -// and updates the minimum price required by the transaction pool. -func (s *Ethereum) StartMining() error { - // If the miner was not running, initialize it - if !s.IsMining() { - // Propagate the initial price point to the transaction pool - s.lock.RLock() - price := s.gasPrice - s.lock.RUnlock() - s.txPool.SetGasTip(price) - - // Configure the local mining address - eb, err := s.Etherbase() - if err != nil { - log.Error("Cannot start mining without etherbase", "err", err) - return fmt.Errorf("etherbase missing: %v", err) - } - var cli *clique.Clique - if c, ok := s.engine.(*clique.Clique); ok { - cli = c - } else if cl, ok := s.engine.(*beacon.Beacon); ok { - if c, ok := cl.InnerEngine().(*clique.Clique); ok { - cli = c - } - } - if cli != nil { - wallet, err := s.accountManager.Find(accounts.Account{Address: eb}) - if wallet == nil || err != nil { - log.Error("Etherbase account unavailable locally", "err", err) - return fmt.Errorf("signer missing: %v", err) - } - cli.Authorize(eb, wallet.SignData) - } - // If mining is started, we can disable the transaction rejection mechanism - // introduced to speed sync times. - s.handler.enableSyncedFeatures() - - go s.miner.Start() - } - return nil -} - -// StopMining terminates the miner, both at the consensus engine level as well as -// at the block creation level. -func (s *Ethereum) StopMining() { - // Update the thread count within the consensus engine - type threaded interface { - SetThreads(threads int) - } - if th, ok := s.engine.(threaded); ok { - th.SetThreads(-1) - } - // Stop the block creating itself - s.miner.Stop() -} - -func (s *Ethereum) IsMining() bool { return s.miner.Mining() } func (s *Ethereum) Miner() *miner.Miner { return s.miner } func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } @@ -490,11 +353,6 @@ func (s *Ethereum) Synced() bool { return s.handler.synced func (s *Ethereum) SetSynced() { s.handler.enableSyncedFeatures() } func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning } func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } -func (s *Ethereum) Merger() *consensus.Merger { return s.merger } -func (s *Ethereum) SyncMode() downloader.SyncMode { - mode, _ := s.handler.chainSync.modeAndLocalHead() - return mode -} func (s *Ethereum) Config() *ethconfig.Config { return s.config } @@ -545,7 +403,6 @@ func (s *Ethereum) Stop() error { s.bloomIndexer.Close() close(s.closeBloomHandler) s.txPool.Close() - s.miner.Close() s.blockchain.Stop() s.engine.Close() @@ -557,3 +414,29 @@ func (s *Ethereum) Stop() error { return nil } + +// SyncMode retrieves the current sync mode, either explicitly set, or derived +// from the chain status. +func (s *Ethereum) SyncMode() downloader.SyncMode { + // If we're in snap sync mode, return that directly + if s.handler.snapSync.Load() { + return downloader.SnapSync + } + // We are probably in full sync, but we might have rewound to before the + // snap sync pivot, check if we should re-enable snap sync. + head := s.blockchain.CurrentBlock() + if pivot := rawdb.ReadLastPivotNumber(s.chainDb); pivot != nil { + if head.Number.Uint64() < *pivot { + return downloader.SnapSync + } + } + // We are in a full sync, but the associated head state is missing. To complete + // the head state, forcefully rerun the snap sync. Note it doesn't mean the + // persistent state is corrupted, just mismatch with the head block. + if !s.blockchain.HasState(head.Root) { + log.Info("Reenabled snap sync as chain is stateless") + return downloader.SnapSync + } + // Nope, we're really full syncing + return downloader.FullSync +} diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 44518612e83f..f549f29dc62b 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -190,21 +190,21 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, pa // attributes. It supports both PayloadAttributesV1 and PayloadAttributesV2. func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { + if params.BeaconRoot != nil { + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("unexpected beacon root")) + } switch api.eth.BlockChain().Config().LatestFork(params.Timestamp) { case forks.Paris: if params.Withdrawals != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals before shanghai")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("withdrawals before shanghai")) } case forks.Shanghai: if params.Withdrawals == nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing withdrawals")) } default: return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV2 must only be called with paris and shanghai payloads")) } - if params.BeaconRoot != nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("unexpected beacon root")) - } } return api.forkchoiceUpdated(update, params, engine.PayloadV2, false) } @@ -213,15 +213,11 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, pa // in the payload attributes. It supports only PayloadAttributesV3. func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { - // TODO(matt): according to https://github.com/ethereum/execution-apis/pull/498, - // payload attributes that are invalid should return error - // engine.InvalidPayloadAttributes. Once hive updates this, we should update - // on our end. if params.Withdrawals == nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing withdrawals")) } if params.BeaconRoot == nil { - return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing beacon root")) + return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing beacon root")) } if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun { return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV3 must only be called for cancun payloads")) @@ -271,12 +267,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl finalized := api.remoteBlocks.get(update.FinalizedBlockHash) // Header advertised via a past newPayload request. Start syncing to it. - // Before we do however, make sure any legacy sync in switched off so we - // don't accidentally have 2 cycles running. - if merger := api.eth.Merger(); !merger.TDDReached() { - merger.ReachTTD() - api.eth.Downloader().Cancel() - } context := []interface{}{"number", header.Number, "hash", header.Hash()} if update.FinalizedBlockHash != (common.Hash{}) { if finalized == nil { @@ -338,9 +328,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl // If the beacon client also advertised a finalized block, mark the local // chain final and completely in PoS mode. if update.FinalizedBlockHash != (common.Hash{}) { - if merger := api.eth.Merger(); !merger.PoSFinalized() { - merger.FinalizePoS() - } // If the finalized block is not in our canonical tree, something is wrong finalBlock := api.eth.BlockChain().GetBlockByHash(update.FinalizedBlockHash) if finalBlock == nil { @@ -488,7 +475,7 @@ func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.Payl // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { if api.eth.BlockChain().Config().IsCancun(api.eth.BlockChain().Config().LondonBlock, params.Timestamp) { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use new payload v2 post-shanghai")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use newPayloadV2 post-cancun")) } if api.eth.BlockChain().Config().LatestFork(params.Timestamp) == forks.Shanghai { if params.Withdrawals == nil { @@ -503,7 +490,7 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil excessBlobGas pre-cancun")) } if params.BlobGasUsed != nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil params.BlobGasUsed pre-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil blobGasUsed pre-cancun")) } return api.newPayload(params, nil, nil) } @@ -517,14 +504,14 @@ func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHas return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil excessBlobGas post-cancun")) } if params.BlobGasUsed == nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil params.BlobGasUsed post-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil blobGasUsed post-cancun")) } if versionedHashes == nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil versionedHashes post-cancun")) } if beaconRoot == nil { - return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil parentBeaconBlockRoot post-cancun")) + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil beaconRoot post-cancun")) } if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun { @@ -580,7 +567,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe // update after legit payload executions. parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return api.delayPayloadImport(block) + return api.delayPayloadImport(block), nil } // We have an existing parent, do some sanity checks to avoid the beacon client // triggering too early @@ -606,7 +593,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe // into the database directly will conflict with the assumptions of snap sync // that it has an empty db that it can fill itself. if api.eth.SyncMode() != downloader.FullSync { - return api.delayPayloadImport(block) + return api.delayPayloadImport(block), nil } if !api.eth.BlockChain().HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { api.remoteBlocks.put(block.Hash(), block.Header()) @@ -624,13 +611,6 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe return api.invalid(err, parent.Header()), nil } - // We've accepted a valid payload from the beacon client. Mark the local - // chain transitions to notify other subsystems (e.g. downloader) of the - // behavioral change. - if merger := api.eth.Merger(); !merger.TDDReached() { - merger.ReachTTD() - api.eth.Downloader().Cancel() - } hash := block.Hash() return engine.PayloadStatusV1{Status: engine.VALID, LatestValidHash: &hash}, nil } @@ -639,11 +619,11 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe // either via a forkchoice update or a sync extension. This method is meant to // be called by the newpayload command when the block seems to be ok, but some // prerequisite prevents it from being processed (e.g. no parent, or snap sync). -func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) delayPayloadImport(block *types.Block) engine.PayloadStatusV1 { // Sanity check that this block's parent is not on a previously invalidated // chain. If it is, mark the block as invalid too. if res := api.checkInvalidAncestor(block.ParentHash(), block.Hash()); res != nil { - return *res, nil + return *res } // Stash the block away for a potential forced forkchoice update to it // at a later time. @@ -655,7 +635,7 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (engine.PayloadS err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()) if err == nil { log.Debug("Payload accepted for sync extension", "number", block.NumberU64(), "hash", block.Hash()) - return engine.PayloadStatusV1{Status: engine.SYNCING}, nil + return engine.PayloadStatusV1{Status: engine.SYNCING} } // Either no beacon sync was started yet, or it rejected the delivered // payload as non-integratable on top of the existing sync. We'll just @@ -672,7 +652,7 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (engine.PayloadS // and cannot afford concurrent out-if-band modifications via imports. log.Warn("Ignoring payload while snap syncing", "number", block.NumberU64(), "hash", block.Hash(), "reason", err) } - return engine.PayloadStatusV1{Status: engine.SYNCING}, nil + return engine.PayloadStatusV1{Status: engine.SYNCING} } // setInvalidAncestor is a callback for the downloader to notify us if a bad block @@ -788,26 +768,23 @@ func (api *ConsensusAPI) heartbeat() { // If there have been no updates for the past while, warn the user // that the beacon client is probably offline - if api.eth.BlockChain().Config().TerminalTotalDifficultyPassed || api.eth.Merger().TDDReached() { - if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout { - offlineLogged = time.Time{} - continue - } - - if time.Since(offlineLogged) > beaconUpdateWarnFrequency { - if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { - if lastTransitionUpdate.IsZero() { - log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") - } else { - log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") - } + if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout { + offlineLogged = time.Time{} + continue + } + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { + if lastTransitionUpdate.IsZero() { + log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") } else { - log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!") + log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") } - offlineLogged = time.Now() + } else { + log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!") } - continue + offlineLogged = time.Now() } + continue } } @@ -879,8 +856,7 @@ func getBody(block *types.Block) *engine.ExecutionPayloadBodyV1 { ) for j, tx := range body.Transactions { - data, _ := tx.MarshalBinary() - txs[j] = hexutil.Bytes(data) + txs[j], _ = tx.MarshalBinary() } // Post-shanghai withdrawals MUST be set to empty slice instead of nil diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 9856118eae3f..a88996744c04 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -262,11 +262,8 @@ func TestInvalidPayloadTimestamp(t *testing.T) { {0, true}, {parent.Time, true}, {parent.Time - 1, true}, - - // TODO (MariusVanDerWijden) following tests are currently broken, - // fixed in upcoming merge-kiln-v2 pr - //{parent.Time() + 1, false}, - //{uint64(time.Now().Unix()) + uint64(time.Minute), false}, + {parent.Time + 1, false}, + {uint64(time.Now().Unix()) + uint64(time.Minute), false}, } for i, test := range tests { @@ -450,7 +447,9 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} + mcfg := miner.DefaultConfig + mcfg.PendingFeeRecipient = testAddr + ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: mcfg} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) @@ -463,7 +462,6 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't import test blocks:", err) } - ethservice.SetEtherbase(testAddr) ethservice.SetSynced() return n, ethservice } @@ -865,7 +863,6 @@ func TestTrickRemoteBlockCache(t *testing.T) { func TestInvalidBloom(t *testing.T) { genesis, preMergeBlocks := generateMergeChain(10, false) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() commonAncestor := ethservice.BlockChain().CurrentBlock() @@ -1047,7 +1044,6 @@ func TestWithdrawals(t *testing.T) { genesis.Config.ShanghaiTime = &time n, ethservice := startEthService(t, genesis, blocks) - ethservice.Merger().ReachTTD() defer n.Close() api := NewConsensusAPI(ethservice) @@ -1165,7 +1161,6 @@ func TestNilWithdrawals(t *testing.T) { genesis.Config.ShanghaiTime = &time n, ethservice := startEthService(t, genesis, blocks) - ethservice.Merger().ReachTTD() defer n.Close() api := NewConsensusAPI(ethservice) @@ -1592,7 +1587,6 @@ func TestParentBeaconBlockRoot(t *testing.T) { genesis.Config.CancunTime = &time n, ethservice := startEthService(t, genesis, blocks) - ethservice.Merger().ReachTTD() defer n.Close() api := NewConsensusAPI(ethservice) diff --git a/eth/catalyst/blsync.go b/eth/catalyst/blsync.go new file mode 100644 index 000000000000..4877cf4c6361 --- /dev/null +++ b/eth/catalyst/blsync.go @@ -0,0 +1,88 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" +) + +// Blsync tracks the head of the beacon chain through the beacon light client +// and drives the local node via ConsensusAPI. +type Blsync struct { + engine *ConsensusAPI + client Client + headCh chan types.ChainHeadEvent + headSub event.Subscription + + quitCh chan struct{} +} + +type Client interface { + SubscribeChainHeadEvent(ch chan<- types.ChainHeadEvent) event.Subscription + Start() + Stop() +} + +// NewBlsync creates a new beacon light syncer. +func NewBlsync(client Client, eth *eth.Ethereum) *Blsync { + return &Blsync{ + engine: newConsensusAPIWithoutHeartbeat(eth), + client: client, + headCh: make(chan types.ChainHeadEvent, 16), + quitCh: make(chan struct{}), + } +} + +// Start starts underlying beacon light client and the sync logic for driving +// the local node. +func (b *Blsync) Start() error { + log.Info("Beacon light sync started") + b.headSub = b.client.SubscribeChainHeadEvent(b.headCh) + go b.client.Start() + + for { + select { + case <-b.quitCh: + return nil + case head := <-b.headCh: + if _, err := b.engine.NewPayloadV2(*head.HeadBlock); err != nil { + log.Error("failed to send new payload", "err", err) + continue + } + update := engine.ForkchoiceStateV1{ + HeadBlockHash: head.HeadBlock.BlockHash, + SafeBlockHash: head.Finalized, //TODO pass finalized or empty hash here? + FinalizedBlockHash: head.Finalized, + } + if _, err := b.engine.ForkchoiceUpdatedV1(update, nil); err != nil { + log.Error("failed to send forkchoice updated", "err", err) + continue + } + } + } +} + +// Stop signals to the light client and syncer to exit. +func (b *Blsync) Stop() error { + b.client.Stop() + close(b.quitCh) + return nil +} diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index 91ac1771d283..4ae60ed4907c 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -18,6 +18,7 @@ package catalyst import ( "crypto/rand" + "crypto/sha256" "errors" "math/big" "sync" @@ -25,7 +26,9 @@ import ( "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -160,14 +163,14 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u SuggestedFeeRecipient: feeRecipient, Withdrawals: withdrawals, Random: random, - }, engine.PayloadV2, true) + BeaconRoot: &common.Hash{}, + }, engine.PayloadV3, true) if err != nil { return err } if fcResponse == engine.STATUS_SYNCING { return errors.New("chain rewind prevented invocation of payload creation") } - envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true) if err != nil { return err @@ -185,8 +188,21 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u } } + // Independently calculate the blob hashes from sidecars. + blobHashes := make([]common.Hash, 0) + if envelope.BlobsBundle != nil { + hasher := sha256.New() + for _, commit := range envelope.BlobsBundle.Commitments { + var c kzg4844.Commitment + if len(commit) != len(c) { + return errors.New("invalid commitment length") + } + copy(c[:], commit) + blobHashes = append(blobHashes, kzg4844.CalcBlobHashV1(hasher, &c)) + } + } // Mark the payload as canon - if _, err = c.engineAPI.NewPayloadV2(*payload); err != nil { + if _, err = c.engineAPI.NewPayloadV3(*payload, blobHashes, &common.Hash{}); err != nil { return err } c.setCurrentState(payload.BlockHash, finalizedHash) @@ -263,7 +279,7 @@ func (c *SimulatedBeacon) Rollback() { // Fork sets the head to the provided hash. func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { - if len(c.eth.TxPool().Pending(nil, nil, nil)) != 0 { + if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { return errors.New("pending block dirty") } parent := c.eth.BlockChain().GetBlockByHash(parentHash) @@ -275,7 +291,7 @@ func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { // AdjustTime creates a new block with an adjusted timestamp. func (c *SimulatedBeacon) AdjustTime(adjustment time.Duration) error { - if len(c.eth.TxPool().Pending(nil, nil, nil)) != 0 { + if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { return errors.New("could not adjust time on non-empty block") } parent := c.eth.BlockChain().CurrentBlock() diff --git a/eth/catalyst/simulated_beacon_test.go b/eth/catalyst/simulated_beacon_test.go index 6fa97ad87a2a..df682b49d96e 100644 --- a/eth/catalyst/simulated_beacon_test.go +++ b/eth/catalyst/simulated_beacon_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" @@ -48,7 +49,7 @@ func startSimulatedBeaconEthService(t *testing.T, genesis *core.Genesis) (*node. t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} + ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: miner.DefaultConfig} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) diff --git a/eth/downloader/api.go b/eth/downloader/api.go index f09122904c4c..90c36afbb5ba 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -38,7 +38,7 @@ type DownloaderAPI struct { uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest } -// NewDownloaderAPI create a new DownloaderAPI. The API has an internal event loop that +// NewDownloaderAPI creates a new DownloaderAPI. The API has an internal event loop that // listens for events from the downloader through the global event mux. In case it receives one of // these events it broadcasts it to all syncing subscriptions that are installed through the // installSyncSubscription channel. @@ -149,8 +149,6 @@ func (api *DownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error notifier.Notify(rpcSub.ID, status) case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index ad664afb5bd1..420a8b147a7f 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -165,15 +165,14 @@ type Config struct { // Clique is allowed for now to live standalone, but ethash is forbidden and can // only exist on already merged networks. func CreateConsensusEngine(config *params.ChainConfig, db ethdb.Database) (consensus.Engine, error) { - // If proof-of-authority is requested, set it up + // Geth v1.14.0 dropped support for non-merged networks in any consensus + // mode. If such a network is requested, reject startup. + if !config.TerminalTotalDifficultyPassed { + return nil, errors.New("only PoS networks are supported, please transition old ones with Geth v1.13.x") + } + // Wrap previously supported consensus engines into their post-merge counterpart if config.Clique != nil { return beacon.New(clique.New(config.Clique, db)), nil } - // If defaulting to proof-of-work, enforce an already merged network since - // we cannot run PoW algorithms anymore, so we cannot even follow a chain - // not coordinated by a beacon node. - if !config.TerminalTotalDifficultyPassed { - return nil, errors.New("ethash is only supported as a historical component of already merged networks") - } return beacon.New(ethash.NewFaker()), nil } diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go deleted file mode 100644 index 126eaaea7fad..000000000000 --- a/eth/fetcher/block_fetcher.go +++ /dev/null @@ -1,939 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package fetcher contains the announcement based header, blocks or transaction synchronisation. -package fetcher - -import ( - "errors" - "math/rand" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/prque" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/trie" -) - -const ( - lightTimeout = time.Millisecond // Time allowance before an announced header is explicitly requested - arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block/transaction is explicitly requested - gatherSlack = 100 * time.Millisecond // Interval used to collate almost-expired announces with fetches - fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block/transaction -) - -const ( - maxUncleDist = 7 // Maximum allowed backward distance from the chain head - maxQueueDist = 32 // Maximum allowed distance from the chain head to queue - hashLimit = 256 // Maximum number of unique blocks or headers a peer may have announced - blockLimit = 64 // Maximum number of unique blocks a peer may have delivered -) - -var ( - blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/in", nil) - blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/announces/out", nil) - blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/drop", nil) - blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/dos", nil) - - blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/in", nil) - blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/broadcasts/out", nil) - blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/drop", nil) - blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/dos", nil) - - headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/headers", nil) - bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/bodies", nil) - - headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/in", nil) - headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/out", nil) - bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/in", nil) - bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/out", nil) -) - -var errTerminated = errors.New("terminated") - -// HeaderRetrievalFn is a callback type for retrieving a header from the local chain. -type HeaderRetrievalFn func(common.Hash) *types.Header - -// blockRetrievalFn is a callback type for retrieving a block from the local chain. -type blockRetrievalFn func(common.Hash) *types.Block - -// headerRequesterFn is a callback type for sending a header retrieval request. -type headerRequesterFn func(common.Hash, chan *eth.Response) (*eth.Request, error) - -// bodyRequesterFn is a callback type for sending a body retrieval request. -type bodyRequesterFn func([]common.Hash, chan *eth.Response) (*eth.Request, error) - -// headerVerifierFn is a callback type to verify a block's header for fast propagation. -type headerVerifierFn func(header *types.Header) error - -// blockBroadcasterFn is a callback type for broadcasting a block to connected peers. -type blockBroadcasterFn func(block *types.Block, propagate bool) - -// chainHeightFn is a callback type to retrieve the current chain height. -type chainHeightFn func() uint64 - -// headersInsertFn is a callback type to insert a batch of headers into the local chain. -type headersInsertFn func(headers []*types.Header) (int, error) - -// chainInsertFn is a callback type to insert a batch of blocks into the local chain. -type chainInsertFn func(types.Blocks) (int, error) - -// peerDropFn is a callback type for dropping a peer detected as malicious. -type peerDropFn func(id string) - -// blockAnnounce is the hash notification of the availability of a new block in the -// network. -type blockAnnounce struct { - hash common.Hash // Hash of the block being announced - number uint64 // Number of the block being announced (0 = unknown | old protocol) - header *types.Header // Header of the block partially reassembled (new protocol) - time time.Time // Timestamp of the announcement - - origin string // Identifier of the peer originating the notification - - fetchHeader headerRequesterFn // Fetcher function to retrieve the header of an announced block - fetchBodies bodyRequesterFn // Fetcher function to retrieve the body of an announced block -} - -// headerFilterTask represents a batch of headers needing fetcher filtering. -type headerFilterTask struct { - peer string // The source peer of block headers - headers []*types.Header // Collection of headers to filter - time time.Time // Arrival time of the headers -} - -// bodyFilterTask represents a batch of block bodies (transactions and uncles) -// needing fetcher filtering. -type bodyFilterTask struct { - peer string // The source peer of block bodies - transactions [][]*types.Transaction // Collection of transactions per block bodies - uncles [][]*types.Header // Collection of uncles per block bodies - time time.Time // Arrival time of the blocks' contents -} - -// blockOrHeaderInject represents a schedules import operation. -type blockOrHeaderInject struct { - origin string - - header *types.Header // Used for light mode fetcher which only cares about header. - block *types.Block // Used for normal mode fetcher which imports full block. -} - -// number returns the block number of the injected object. -func (inject *blockOrHeaderInject) number() uint64 { - if inject.header != nil { - return inject.header.Number.Uint64() - } - return inject.block.NumberU64() -} - -// number returns the block hash of the injected object. -func (inject *blockOrHeaderInject) hash() common.Hash { - if inject.header != nil { - return inject.header.Hash() - } - return inject.block.Hash() -} - -// BlockFetcher is responsible for accumulating block announcements from various peers -// and scheduling them for retrieval. -type BlockFetcher struct { - light bool // The indicator whether it's a light fetcher or normal one. - - // Various event channels - notify chan *blockAnnounce - inject chan *blockOrHeaderInject - - headerFilter chan chan *headerFilterTask - bodyFilter chan chan *bodyFilterTask - - done chan common.Hash - quit chan struct{} - - // Announce states - announces map[string]int // Per peer blockAnnounce counts to prevent memory exhaustion - announced map[common.Hash][]*blockAnnounce // Announced blocks, scheduled for fetching - fetching map[common.Hash]*blockAnnounce // Announced blocks, currently fetching - fetched map[common.Hash][]*blockAnnounce // Blocks with headers fetched, scheduled for body retrieval - completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing - - // Block cache - queue *prque.Prque[int64, *blockOrHeaderInject] // Queue containing the import operations (block number sorted) - queues map[string]int // Per peer block counts to prevent memory exhaustion - queued map[common.Hash]*blockOrHeaderInject // Set of already queued blocks (to dedup imports) - - // Callbacks - getHeader HeaderRetrievalFn // Retrieves a header from the local chain - getBlock blockRetrievalFn // Retrieves a block from the local chain - verifyHeader headerVerifierFn // Checks if a block's headers have a valid proof of work - broadcastBlock blockBroadcasterFn // Broadcasts a block to connected peers - chainHeight chainHeightFn // Retrieves the current chain's height - insertHeaders headersInsertFn // Injects a batch of headers into the chain - insertChain chainInsertFn // Injects a batch of blocks into the chain - dropPeer peerDropFn // Drops a peer for misbehaving - - // Testing hooks - announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the blockAnnounce list - queueChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a block from the import queue - fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch - completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62) - importedHook func(*types.Header, *types.Block) // Method to call upon successful header or block import (both eth/61 and eth/62) -} - -// NewBlockFetcher creates a block fetcher to retrieve blocks based on hash announcements. -func NewBlockFetcher(light bool, getHeader HeaderRetrievalFn, getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertHeaders headersInsertFn, insertChain chainInsertFn, dropPeer peerDropFn) *BlockFetcher { - return &BlockFetcher{ - light: light, - notify: make(chan *blockAnnounce), - inject: make(chan *blockOrHeaderInject), - headerFilter: make(chan chan *headerFilterTask), - bodyFilter: make(chan chan *bodyFilterTask), - done: make(chan common.Hash), - quit: make(chan struct{}), - announces: make(map[string]int), - announced: make(map[common.Hash][]*blockAnnounce), - fetching: make(map[common.Hash]*blockAnnounce), - fetched: make(map[common.Hash][]*blockAnnounce), - completing: make(map[common.Hash]*blockAnnounce), - queue: prque.New[int64, *blockOrHeaderInject](nil), - queues: make(map[string]int), - queued: make(map[common.Hash]*blockOrHeaderInject), - getHeader: getHeader, - getBlock: getBlock, - verifyHeader: verifyHeader, - broadcastBlock: broadcastBlock, - chainHeight: chainHeight, - insertHeaders: insertHeaders, - insertChain: insertChain, - dropPeer: dropPeer, - } -} - -// Start boots up the announcement based synchroniser, accepting and processing -// hash notifications and block fetches until termination requested. -func (f *BlockFetcher) Start() { - go f.loop() -} - -// Stop terminates the announcement based synchroniser, canceling all pending -// operations. -func (f *BlockFetcher) Stop() { - close(f.quit) -} - -// Notify announces the fetcher of the potential availability of a new block in -// the network. -func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, - headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error { - block := &blockAnnounce{ - hash: hash, - number: number, - time: time, - origin: peer, - fetchHeader: headerFetcher, - fetchBodies: bodyFetcher, - } - select { - case f.notify <- block: - return nil - case <-f.quit: - return errTerminated - } -} - -// Enqueue tries to fill gaps the fetcher's future import queue. -func (f *BlockFetcher) Enqueue(peer string, block *types.Block) error { - op := &blockOrHeaderInject{ - origin: peer, - block: block, - } - select { - case f.inject <- op: - return nil - case <-f.quit: - return errTerminated - } -} - -// FilterHeaders extracts all the headers that were explicitly requested by the fetcher, -// returning those that should be handled differently. -func (f *BlockFetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header { - log.Trace("Filtering headers", "peer", peer, "headers", len(headers)) - - // Send the filter channel to the fetcher - filter := make(chan *headerFilterTask) - - select { - case f.headerFilter <- filter: - case <-f.quit: - return nil - } - // Request the filtering of the header list - select { - case filter <- &headerFilterTask{peer: peer, headers: headers, time: time}: - case <-f.quit: - return nil - } - // Retrieve the headers remaining after filtering - select { - case task := <-filter: - return task.headers - case <-f.quit: - return nil - } -} - -// FilterBodies extracts all the block bodies that were explicitly requested by -// the fetcher, returning those that should be handled differently. -func (f *BlockFetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) { - log.Trace("Filtering bodies", "peer", peer, "txs", len(transactions), "uncles", len(uncles)) - - // Send the filter channel to the fetcher - filter := make(chan *bodyFilterTask) - - select { - case f.bodyFilter <- filter: - case <-f.quit: - return nil, nil - } - // Request the filtering of the body list - select { - case filter <- &bodyFilterTask{peer: peer, transactions: transactions, uncles: uncles, time: time}: - case <-f.quit: - return nil, nil - } - // Retrieve the bodies remaining after filtering - select { - case task := <-filter: - return task.transactions, task.uncles - case <-f.quit: - return nil, nil - } -} - -// Loop is the main fetcher loop, checking and processing various notification -// events. -func (f *BlockFetcher) loop() { - // Iterate the block fetching until a quit is requested - var ( - fetchTimer = time.NewTimer(0) - completeTimer = time.NewTimer(0) - ) - <-fetchTimer.C // clear out the channel - <-completeTimer.C - defer fetchTimer.Stop() - defer completeTimer.Stop() - - for { - // Clean up any expired block fetches - for hash, announce := range f.fetching { - if time.Since(announce.time) > fetchTimeout { - f.forgetHash(hash) - } - } - // Import any queued blocks that could potentially fit - height := f.chainHeight() - for !f.queue.Empty() { - op := f.queue.PopItem() - hash := op.hash() - if f.queueChangeHook != nil { - f.queueChangeHook(hash, false) - } - // If too high up the chain or phase, continue later - number := op.number() - if number > height+1 { - f.queue.Push(op, -int64(number)) - if f.queueChangeHook != nil { - f.queueChangeHook(hash, true) - } - break - } - // Otherwise if fresh and still unknown, try and import - if (number+maxUncleDist < height) || (f.light && f.getHeader(hash) != nil) || (!f.light && f.getBlock(hash) != nil) { - f.forgetBlock(hash) - continue - } - if f.light { - f.importHeaders(op.origin, op.header) - } else { - f.importBlocks(op.origin, op.block) - } - } - // Wait for an outside event to occur - select { - case <-f.quit: - // BlockFetcher terminating, abort all operations - return - - case notification := <-f.notify: - // A block was announced, make sure the peer isn't DOSing us - blockAnnounceInMeter.Mark(1) - - count := f.announces[notification.origin] + 1 - if count > hashLimit { - log.Debug("Peer exceeded outstanding announces", "peer", notification.origin, "limit", hashLimit) - blockAnnounceDOSMeter.Mark(1) - break - } - if notification.number == 0 { - break - } - // If we have a valid block number, check that it's potentially useful - if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { - log.Debug("Peer discarded announcement", "peer", notification.origin, "number", notification.number, "hash", notification.hash, "distance", dist) - blockAnnounceDropMeter.Mark(1) - break - } - // All is well, schedule the announce if block's not yet downloading - if _, ok := f.fetching[notification.hash]; ok { - break - } - if _, ok := f.completing[notification.hash]; ok { - break - } - f.announces[notification.origin] = count - f.announced[notification.hash] = append(f.announced[notification.hash], notification) - if f.announceChangeHook != nil && len(f.announced[notification.hash]) == 1 { - f.announceChangeHook(notification.hash, true) - } - if len(f.announced) == 1 { - f.rescheduleFetch(fetchTimer) - } - - case op := <-f.inject: - // A direct block insertion was requested, try and fill any pending gaps - blockBroadcastInMeter.Mark(1) - - // Now only direct block injection is allowed, drop the header injection - // here silently if we receive. - if f.light { - continue - } - f.enqueue(op.origin, nil, op.block) - - case hash := <-f.done: - // A pending import finished, remove all traces of the notification - f.forgetHash(hash) - f.forgetBlock(hash) - - case <-fetchTimer.C: - // At least one block's timer ran out, check for needing retrieval - request := make(map[string][]common.Hash) - - for hash, announces := range f.announced { - // In current LES protocol(les2/les3), only header announce is - // available, no need to wait too much time for header broadcast. - timeout := arriveTimeout - gatherSlack - if f.light { - timeout = 0 - } - if time.Since(announces[0].time) > timeout { - // Pick a random peer to retrieve from, reset all others - announce := announces[rand.Intn(len(announces))] - f.forgetHash(hash) - - // If the block still didn't arrive, queue for fetching - if (f.light && f.getHeader(hash) == nil) || (!f.light && f.getBlock(hash) == nil) { - request[announce.origin] = append(request[announce.origin], hash) - f.fetching[hash] = announce - } - } - } - // Send out all block header requests - for peer, hashes := range request { - log.Trace("Fetching scheduled headers", "peer", peer, "list", hashes) - - // Create a closure of the fetch and schedule in on a new thread - fetchHeader, hashes := f.fetching[hashes[0]].fetchHeader, hashes - go func(peer string) { - if f.fetchingHook != nil { - f.fetchingHook(hashes) - } - for _, hash := range hashes { - headerFetchMeter.Mark(1) - go func(hash common.Hash) { - resCh := make(chan *eth.Response) - - req, err := fetchHeader(hash, resCh) - if err != nil { - return // Legacy code, yolo - } - defer req.Close() - - timeout := time.NewTimer(2 * fetchTimeout) // 2x leeway before dropping the peer - defer timeout.Stop() - - select { - case res := <-resCh: - res.Done <- nil - f.FilterHeaders(peer, *res.Res.(*eth.BlockHeadersRequest), time.Now()) - - case <-timeout.C: - // The peer didn't respond in time. The request - // was already rescheduled at this point, we were - // waiting for a catchup. With an unresponsive - // peer however, it's a protocol violation. - f.dropPeer(peer) - } - }(hash) - } - }(peer) - } - // Schedule the next fetch if blocks are still pending - f.rescheduleFetch(fetchTimer) - - case <-completeTimer.C: - // At least one header's timer ran out, retrieve everything - request := make(map[string][]common.Hash) - - for hash, announces := range f.fetched { - // Pick a random peer to retrieve from, reset all others - announce := announces[rand.Intn(len(announces))] - f.forgetHash(hash) - - // If the block still didn't arrive, queue for completion - if f.getBlock(hash) == nil { - request[announce.origin] = append(request[announce.origin], hash) - f.completing[hash] = announce - } - } - // Send out all block body requests - for peer, hashes := range request { - log.Trace("Fetching scheduled bodies", "peer", peer, "list", hashes) - - // Create a closure of the fetch and schedule in on a new thread - if f.completingHook != nil { - f.completingHook(hashes) - } - fetchBodies := f.completing[hashes[0]].fetchBodies - bodyFetchMeter.Mark(int64(len(hashes))) - - go func(peer string, hashes []common.Hash) { - resCh := make(chan *eth.Response) - - req, err := fetchBodies(hashes, resCh) - if err != nil { - return // Legacy code, yolo - } - defer req.Close() - - timeout := time.NewTimer(2 * fetchTimeout) // 2x leeway before dropping the peer - defer timeout.Stop() - - select { - case res := <-resCh: - res.Done <- nil - // Ignoring withdrawals here, since the block fetcher is not used post-merge. - txs, uncles, _ := res.Res.(*eth.BlockBodiesResponse).Unpack() - f.FilterBodies(peer, txs, uncles, time.Now()) - - case <-timeout.C: - // The peer didn't respond in time. The request - // was already rescheduled at this point, we were - // waiting for a catchup. With an unresponsive - // peer however, it's a protocol violation. - f.dropPeer(peer) - } - }(peer, hashes) - } - // Schedule the next fetch if blocks are still pending - f.rescheduleComplete(completeTimer) - - case filter := <-f.headerFilter: - // Headers arrived from a remote peer. Extract those that were explicitly - // requested by the fetcher, and return everything else so it's delivered - // to other parts of the system. - var task *headerFilterTask - select { - case task = <-filter: - case <-f.quit: - return - } - headerFilterInMeter.Mark(int64(len(task.headers))) - - // Split the batch of headers into unknown ones (to return to the caller), - // known incomplete ones (requiring body retrievals) and completed blocks. - unknown, incomplete, complete, lightHeaders := []*types.Header{}, []*blockAnnounce{}, []*types.Block{}, []*blockAnnounce{} - for _, header := range task.headers { - hash := header.Hash() - - // Filter fetcher-requested headers from other synchronisation algorithms - if announce := f.fetching[hash]; announce != nil && announce.origin == task.peer && f.fetched[hash] == nil && f.completing[hash] == nil && f.queued[hash] == nil { - // If the delivered header does not match the promised number, drop the announcer - if header.Number.Uint64() != announce.number { - log.Trace("Invalid block number fetched", "peer", announce.origin, "hash", header.Hash(), "announced", announce.number, "provided", header.Number) - f.dropPeer(announce.origin) - f.forgetHash(hash) - continue - } - // Collect all headers only if we are running in light - // mode and the headers are not imported by other means. - if f.light { - if f.getHeader(hash) == nil { - announce.header = header - lightHeaders = append(lightHeaders, announce) - } - f.forgetHash(hash) - continue - } - // Only keep if not imported by other means - if f.getBlock(hash) == nil { - announce.header = header - announce.time = task.time - - // If the block is empty (header only), short circuit into the final import queue - if header.TxHash == types.EmptyTxsHash && header.UncleHash == types.EmptyUncleHash { - log.Trace("Block empty, skipping body retrieval", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) - - block := types.NewBlockWithHeader(header) - block.ReceivedAt = task.time - - complete = append(complete, block) - f.completing[hash] = announce - continue - } - // Otherwise add to the list of blocks needing completion - incomplete = append(incomplete, announce) - } else { - log.Trace("Block already imported, discarding header", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) - f.forgetHash(hash) - } - } else { - // BlockFetcher doesn't know about it, add to the return list - unknown = append(unknown, header) - } - } - headerFilterOutMeter.Mark(int64(len(unknown))) - select { - case filter <- &headerFilterTask{headers: unknown, time: task.time}: - case <-f.quit: - return - } - // Schedule the retrieved headers for body completion - for _, announce := range incomplete { - hash := announce.header.Hash() - if _, ok := f.completing[hash]; ok { - continue - } - f.fetched[hash] = append(f.fetched[hash], announce) - if len(f.fetched) == 1 { - f.rescheduleComplete(completeTimer) - } - } - // Schedule the header for light fetcher import - for _, announce := range lightHeaders { - f.enqueue(announce.origin, announce.header, nil) - } - // Schedule the header-only blocks for import - for _, block := range complete { - if announce := f.completing[block.Hash()]; announce != nil { - f.enqueue(announce.origin, nil, block) - } - } - - case filter := <-f.bodyFilter: - // Block bodies arrived, extract any explicitly requested blocks, return the rest - var task *bodyFilterTask - select { - case task = <-filter: - case <-f.quit: - return - } - bodyFilterInMeter.Mark(int64(len(task.transactions))) - blocks := []*types.Block{} - // abort early if there's nothing explicitly requested - if len(f.completing) > 0 { - for i := 0; i < len(task.transactions) && i < len(task.uncles); i++ { - // Match up a body to any possible completion request - var ( - matched = false - uncleHash common.Hash // calculated lazily and reused - txnHash common.Hash // calculated lazily and reused - ) - for hash, announce := range f.completing { - if f.queued[hash] != nil || announce.origin != task.peer { - continue - } - if uncleHash == (common.Hash{}) { - uncleHash = types.CalcUncleHash(task.uncles[i]) - } - if uncleHash != announce.header.UncleHash { - continue - } - if txnHash == (common.Hash{}) { - txnHash = types.DeriveSha(types.Transactions(task.transactions[i]), trie.NewStackTrie(nil)) - } - if txnHash != announce.header.TxHash { - continue - } - // Mark the body matched, reassemble if still unknown - matched = true - if f.getBlock(hash) == nil { - block := types.NewBlockWithHeader(announce.header).WithBody(task.transactions[i], task.uncles[i]) - block.ReceivedAt = task.time - blocks = append(blocks, block) - } else { - f.forgetHash(hash) - } - } - if matched { - task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) - task.uncles = append(task.uncles[:i], task.uncles[i+1:]...) - i-- - continue - } - } - } - bodyFilterOutMeter.Mark(int64(len(task.transactions))) - select { - case filter <- task: - case <-f.quit: - return - } - // Schedule the retrieved blocks for ordered import - for _, block := range blocks { - if announce := f.completing[block.Hash()]; announce != nil { - f.enqueue(announce.origin, nil, block) - } - } - } - } -} - -// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. -func (f *BlockFetcher) rescheduleFetch(fetch *time.Timer) { - // Short circuit if no blocks are announced - if len(f.announced) == 0 { - return - } - // Schedule announcement retrieval quickly for light mode - // since server won't send any headers to client. - if f.light { - fetch.Reset(lightTimeout) - return - } - // Otherwise find the earliest expiring announcement - earliest := time.Now() - for _, announces := range f.announced { - if earliest.After(announces[0].time) { - earliest = announces[0].time - } - } - fetch.Reset(arriveTimeout - time.Since(earliest)) -} - -// rescheduleComplete resets the specified completion timer to the next fetch timeout. -func (f *BlockFetcher) rescheduleComplete(complete *time.Timer) { - // Short circuit if no headers are fetched - if len(f.fetched) == 0 { - return - } - // Otherwise find the earliest expiring announcement - earliest := time.Now() - for _, announces := range f.fetched { - if earliest.After(announces[0].time) { - earliest = announces[0].time - } - } - complete.Reset(gatherSlack - time.Since(earliest)) -} - -// enqueue schedules a new header or block import operation, if the component -// to be imported has not yet been seen. -func (f *BlockFetcher) enqueue(peer string, header *types.Header, block *types.Block) { - var ( - hash common.Hash - number uint64 - ) - if header != nil { - hash, number = header.Hash(), header.Number.Uint64() - } else { - hash, number = block.Hash(), block.NumberU64() - } - // Ensure the peer isn't DOSing us - count := f.queues[peer] + 1 - if count > blockLimit { - log.Debug("Discarded delivered header or block, exceeded allowance", "peer", peer, "number", number, "hash", hash, "limit", blockLimit) - blockBroadcastDOSMeter.Mark(1) - f.forgetHash(hash) - return - } - // Discard any past or too distant blocks - if dist := int64(number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { - log.Debug("Discarded delivered header or block, too far away", "peer", peer, "number", number, "hash", hash, "distance", dist) - blockBroadcastDropMeter.Mark(1) - f.forgetHash(hash) - return - } - // Schedule the block for future importing - if _, ok := f.queued[hash]; !ok { - op := &blockOrHeaderInject{origin: peer} - if header != nil { - op.header = header - } else { - op.block = block - } - f.queues[peer] = count - f.queued[hash] = op - f.queue.Push(op, -int64(number)) - if f.queueChangeHook != nil { - f.queueChangeHook(hash, true) - } - log.Debug("Queued delivered header or block", "peer", peer, "number", number, "hash", hash, "queued", f.queue.Size()) - } -} - -// importHeaders spawns a new goroutine to run a header insertion into the chain. -// If the header's number is at the same height as the current import phase, it -// updates the phase states accordingly. -func (f *BlockFetcher) importHeaders(peer string, header *types.Header) { - hash := header.Hash() - log.Debug("Importing propagated header", "peer", peer, "number", header.Number, "hash", hash) - - go func() { - defer func() { f.done <- hash }() - // If the parent's unknown, abort insertion - parent := f.getHeader(header.ParentHash) - if parent == nil { - log.Debug("Unknown parent of propagated header", "peer", peer, "number", header.Number, "hash", hash, "parent", header.ParentHash) - return - } - // Validate the header and if something went wrong, drop the peer - if err := f.verifyHeader(header); err != nil && err != consensus.ErrFutureBlock { - log.Debug("Propagated header verification failed", "peer", peer, "number", header.Number, "hash", hash, "err", err) - f.dropPeer(peer) - return - } - // Run the actual import and log any issues - if _, err := f.insertHeaders([]*types.Header{header}); err != nil { - log.Debug("Propagated header import failed", "peer", peer, "number", header.Number, "hash", hash, "err", err) - return - } - // Invoke the testing hook if needed - if f.importedHook != nil { - f.importedHook(header, nil) - } - }() -} - -// importBlocks spawns a new goroutine to run a block insertion into the chain. If the -// block's number is at the same height as the current import phase, it updates -// the phase states accordingly. -func (f *BlockFetcher) importBlocks(peer string, block *types.Block) { - hash := block.Hash() - - // Run the import on a new thread - log.Debug("Importing propagated block", "peer", peer, "number", block.Number(), "hash", hash) - go func() { - defer func() { f.done <- hash }() - - // If the parent's unknown, abort insertion - parent := f.getBlock(block.ParentHash()) - if parent == nil { - log.Debug("Unknown parent of propagated block", "peer", peer, "number", block.Number(), "hash", hash, "parent", block.ParentHash()) - return - } - // Quickly validate the header and propagate the block if it passes - switch err := f.verifyHeader(block.Header()); err { - case nil: - // All ok, quickly propagate to our peers - blockBroadcastOutTimer.UpdateSince(block.ReceivedAt) - go f.broadcastBlock(block, true) - - case consensus.ErrFutureBlock: - // Weird future block, don't fail, but neither propagate - - default: - // Something went very wrong, drop the peer - log.Debug("Propagated block verification failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err) - f.dropPeer(peer) - return - } - // Run the actual import and log any issues - if _, err := f.insertChain(types.Blocks{block}); err != nil { - log.Debug("Propagated block import failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err) - return - } - // If import succeeded, broadcast the block - blockAnnounceOutTimer.UpdateSince(block.ReceivedAt) - go f.broadcastBlock(block, false) - - // Invoke the testing hook if needed - if f.importedHook != nil { - f.importedHook(nil, block) - } - }() -} - -// forgetHash removes all traces of a block announcement from the fetcher's -// internal state. -func (f *BlockFetcher) forgetHash(hash common.Hash) { - // Remove all pending announces and decrement DOS counters - if announceMap, ok := f.announced[hash]; ok { - for _, announce := range announceMap { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - } - delete(f.announced, hash) - if f.announceChangeHook != nil { - f.announceChangeHook(hash, false) - } - } - // Remove any pending fetches and decrement the DOS counters - if announce := f.fetching[hash]; announce != nil { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - delete(f.fetching, hash) - } - - // Remove any pending completion requests and decrement the DOS counters - for _, announce := range f.fetched[hash] { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - } - delete(f.fetched, hash) - - // Remove any pending completions and decrement the DOS counters - if announce := f.completing[hash]; announce != nil { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - delete(f.completing, hash) - } -} - -// forgetBlock removes all traces of a queued block from the fetcher's internal -// state. -func (f *BlockFetcher) forgetBlock(hash common.Hash) { - if insert := f.queued[hash]; insert != nil { - f.queues[insert.origin]-- - if f.queues[insert.origin] == 0 { - delete(f.queues, insert.origin) - } - delete(f.queued, hash) - } -} diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go deleted file mode 100644 index cb7cbaf79edc..000000000000 --- a/eth/fetcher/block_fetcher_test.go +++ /dev/null @@ -1,949 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package fetcher - -import ( - "errors" - "math/big" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/triedb" -) - -var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - gspec = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: types.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - genesis = gspec.MustCommit(testdb, triedb.NewDatabase(testdb, triedb.HashDefaults)) - unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) -) - -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { - blocks, _ := core.GenerateChain(gspec.Config, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { - block.SetCoinbase(common.Address{seed}) - - // If the block number is multiple of 3, send a bonus transaction to the miner - if parent == genesis && i%3 == 0 { - signer := types.MakeSigner(params.TestChainConfig, block.Number(), block.Timestamp()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) - if err != nil { - panic(err) - } - block.AddTx(tx) - } - // If the block number is a multiple of 5, add a bonus uncle to the block - if i > 0 && i%5 == 0 { - block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 2).Hash(), Number: big.NewInt(int64(i - 1))}) - } - }) - hashes := make([]common.Hash, n+1) - hashes[len(hashes)-1] = parent.Hash() - blockm := make(map[common.Hash]*types.Block, n+1) - blockm[parent.Hash()] = parent - for i, b := range blocks { - hashes[len(hashes)-i-2] = b.Hash() - blockm[b.Hash()] = b - } - return hashes, blockm -} - -// fetcherTester is a test simulator for mocking out local block chain. -type fetcherTester struct { - fetcher *BlockFetcher - - hashes []common.Hash // Hash chain belonging to the tester - headers map[common.Hash]*types.Header // Headers belonging to the tester - blocks map[common.Hash]*types.Block // Blocks belonging to the tester - drops map[string]bool // Map of peers dropped by the fetcher - - lock sync.RWMutex -} - -// newTester creates a new fetcher test mocker. -func newTester(light bool) *fetcherTester { - tester := &fetcherTester{ - hashes: []common.Hash{genesis.Hash()}, - headers: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, - blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, - drops: make(map[string]bool), - } - tester.fetcher = NewBlockFetcher(light, tester.getHeader, tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertHeaders, tester.insertChain, tester.dropPeer) - tester.fetcher.Start() - - return tester -} - -// getHeader retrieves a header from the tester's block chain. -func (f *fetcherTester) getHeader(hash common.Hash) *types.Header { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.headers[hash] -} - -// getBlock retrieves a block from the tester's block chain. -func (f *fetcherTester) getBlock(hash common.Hash) *types.Block { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.blocks[hash] -} - -// verifyHeader is a nop placeholder for the block header verification. -func (f *fetcherTester) verifyHeader(header *types.Header) error { - return nil -} - -// broadcastBlock is a nop placeholder for the block broadcasting. -func (f *fetcherTester) broadcastBlock(block *types.Block, propagate bool) { -} - -// chainHeight retrieves the current height (block number) of the chain. -func (f *fetcherTester) chainHeight() uint64 { - f.lock.RLock() - defer f.lock.RUnlock() - - if f.fetcher.light { - return f.headers[f.hashes[len(f.hashes)-1]].Number.Uint64() - } - return f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() -} - -// insertChain injects a new headers into the simulated chain. -func (f *fetcherTester) insertHeaders(headers []*types.Header) (int, error) { - f.lock.Lock() - defer f.lock.Unlock() - - for i, header := range headers { - // Make sure the parent in known - if _, ok := f.headers[header.ParentHash]; !ok { - return i, errors.New("unknown parent") - } - // Discard any new blocks if the same height already exists - if header.Number.Uint64() <= f.headers[f.hashes[len(f.hashes)-1]].Number.Uint64() { - return i, nil - } - // Otherwise build our current chain - f.hashes = append(f.hashes, header.Hash()) - f.headers[header.Hash()] = header - } - return 0, nil -} - -// insertChain injects a new blocks into the simulated chain. -func (f *fetcherTester) insertChain(blocks types.Blocks) (int, error) { - f.lock.Lock() - defer f.lock.Unlock() - - for i, block := range blocks { - // Make sure the parent in known - if _, ok := f.blocks[block.ParentHash()]; !ok { - return i, errors.New("unknown parent") - } - // Discard any new blocks if the same height already exists - if block.NumberU64() <= f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() { - return i, nil - } - // Otherwise build our current chain - f.hashes = append(f.hashes, block.Hash()) - f.blocks[block.Hash()] = block - } - return 0, nil -} - -// dropPeer is an emulator for the peer removal, simply accumulating the various -// peers dropped by the fetcher. -func (f *fetcherTester) dropPeer(peer string) { - f.lock.Lock() - defer f.lock.Unlock() - - f.drops[peer] = true -} - -// makeHeaderFetcher retrieves a block header fetcher associated with a simulated peer. -func (f *fetcherTester) makeHeaderFetcher(peer string, blocks map[common.Hash]*types.Block, drift time.Duration) headerRequesterFn { - closure := make(map[common.Hash]*types.Block) - for hash, block := range blocks { - closure[hash] = block - } - // Create a function that return a header from the closure - return func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - // Gather the blocks to return - headers := make([]*types.Header, 0, 1) - if block, ok := closure[hash]; ok { - headers = append(headers, block.Header()) - } - // Return on a new thread - req := ð.Request{ - Peer: peer, - } - res := ð.Response{ - Req: req, - Res: (*eth.BlockHeadersRequest)(&headers), - Time: drift, - Done: make(chan error, 1), // Ignore the returned status - } - go func() { - sink <- res - }() - return req, nil - } -} - -// makeBodyFetcher retrieves a block body fetcher associated with a simulated peer. -func (f *fetcherTester) makeBodyFetcher(peer string, blocks map[common.Hash]*types.Block, drift time.Duration) bodyRequesterFn { - closure := make(map[common.Hash]*types.Block) - for hash, block := range blocks { - closure[hash] = block - } - // Create a function that returns blocks from the closure - return func(hashes []common.Hash, sink chan *eth.Response) (*eth.Request, error) { - // Gather the block bodies to return - transactions := make([][]*types.Transaction, 0, len(hashes)) - uncles := make([][]*types.Header, 0, len(hashes)) - - for _, hash := range hashes { - if block, ok := closure[hash]; ok { - transactions = append(transactions, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } - } - // Return on a new thread - bodies := make([]*eth.BlockBody, len(transactions)) - for i, txs := range transactions { - bodies[i] = ð.BlockBody{ - Transactions: txs, - Uncles: uncles[i], - } - } - req := ð.Request{ - Peer: peer, - } - res := ð.Response{ - Req: req, - Res: (*eth.BlockBodiesResponse)(&bodies), - Time: drift, - Done: make(chan error, 1), // Ignore the returned status - } - go func() { - sink <- res - }() - return req, nil - } -} - -// verifyFetchingEvent verifies that one single event arrive on a fetching channel. -func verifyFetchingEvent(t *testing.T, fetching chan []common.Hash, arrive bool) { - t.Helper() - - if arrive { - select { - case <-fetching: - case <-time.After(time.Second): - t.Fatalf("fetching timeout") - } - } else { - select { - case <-fetching: - t.Fatalf("fetching invoked") - case <-time.After(10 * time.Millisecond): - } - } -} - -// verifyCompletingEvent verifies that one single event arrive on an completing channel. -func verifyCompletingEvent(t *testing.T, completing chan []common.Hash, arrive bool) { - t.Helper() - - if arrive { - select { - case <-completing: - case <-time.After(time.Second): - t.Fatalf("completing timeout") - } - } else { - select { - case <-completing: - t.Fatalf("completing invoked") - case <-time.After(10 * time.Millisecond): - } - } -} - -// verifyImportEvent verifies that one single event arrive on an import channel. -func verifyImportEvent(t *testing.T, imported chan interface{}, arrive bool) { - t.Helper() - - if arrive { - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("import timeout") - } - } else { - select { - case <-imported: - t.Fatalf("import invoked") - case <-time.After(20 * time.Millisecond): - } - } -} - -// verifyImportCount verifies that exactly count number of events arrive on an -// import hook channel. -func verifyImportCount(t *testing.T, imported chan interface{}, count int) { - t.Helper() - - for i := 0; i < count; i++ { - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("block %d: import timeout", i+1) - } - } - verifyImportDone(t, imported) -} - -// verifyImportDone verifies that no more events are arriving on an import channel. -func verifyImportDone(t *testing.T, imported chan interface{}) { - t.Helper() - - select { - case <-imported: - t.Fatalf("extra block imported") - case <-time.After(50 * time.Millisecond): - } -} - -// verifyChainHeight verifies the chain height is as expected. -func verifyChainHeight(t *testing.T, fetcher *fetcherTester, height uint64) { - t.Helper() - - if fetcher.chainHeight() != height { - t.Fatalf("chain height mismatch, got %d, want %d", fetcher.chainHeight(), height) - } -} - -// Tests that a fetcher accepts block/header announcements and initiates retrievals -// for them, successfully importing into the local chain. -func TestFullSequentialAnnouncements(t *testing.T) { testSequentialAnnouncements(t, false) } -func TestLightSequentialAnnouncements(t *testing.T) { testSequentialAnnouncements(t, true) } - -func testSequentialAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - tester := newTester(light) - defer tester.fetcher.Stop() - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks until all are imported - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that if blocks are announced by multiple peers (or even the same buggy -// peer), they will only get downloaded at most once. -func TestFullConcurrentAnnouncements(t *testing.T) { testConcurrentAnnouncements(t, false) } -func TestLightConcurrentAnnouncements(t *testing.T) { testConcurrentAnnouncements(t, true) } - -func testConcurrentAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - // Assemble a tester with a built in counter for the requests - tester := newTester(light) - firstHeaderFetcher := tester.makeHeaderFetcher("first", blocks, -gatherSlack) - firstBodyFetcher := tester.makeBodyFetcher("first", blocks, 0) - secondHeaderFetcher := tester.makeHeaderFetcher("second", blocks, -gatherSlack) - secondBodyFetcher := tester.makeBodyFetcher("second", blocks, 0) - - var counter atomic.Uint32 - firstHeaderWrapper := func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - counter.Add(1) - return firstHeaderFetcher(hash, sink) - } - secondHeaderWrapper := func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - counter.Add(1) - return secondHeaderFetcher(hash, sink) - } - // Iteratively announce blocks until all are imported - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), firstHeaderWrapper, firstBodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), secondHeaderWrapper, secondBodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), secondHeaderWrapper, secondBodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) - - // Make sure no blocks were retrieved twice - if c := int(counter.Load()); c != targetBlocks { - t.Fatalf("retrieval count mismatch: have %v, want %v", c, targetBlocks) - } - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that announcements arriving while a previous is being fetched still -// results in a valid import. -func TestFullOverlappingAnnouncements(t *testing.T) { testOverlappingAnnouncements(t, false) } -func TestLightOverlappingAnnouncements(t *testing.T) { testOverlappingAnnouncements(t, true) } - -func testOverlappingAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, but overlap them continuously - overlap := 16 - imported := make(chan interface{}, len(hashes)-1) - for i := 0; i < overlap; i++ { - imported <- nil - } - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("block %d: import timeout", len(hashes)-i) - } - } - // Wait for all the imports to complete and check count - verifyImportCount(t, imported, overlap) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that announces already being retrieved will not be duplicated. -func TestFullPendingDeduplication(t *testing.T) { testPendingDeduplication(t, false) } -func TestLightPendingDeduplication(t *testing.T) { testPendingDeduplication(t, true) } - -func testPendingDeduplication(t *testing.T, light bool) { - // Create a hash and corresponding block - hashes, blocks := makeChain(1, 0, genesis) - - // Assemble a tester with a built in counter and delayed fetcher - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("repeater", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("repeater", blocks, 0) - - delay := 50 * time.Millisecond - var counter atomic.Uint32 - headerWrapper := func(hash common.Hash, sink chan *eth.Response) (*eth.Request, error) { - counter.Add(1) - - // Simulate a long running fetch - resink := make(chan *eth.Response) - req, err := headerFetcher(hash, resink) - if err == nil { - go func() { - res := <-resink - time.Sleep(delay) - sink <- res - }() - } - return req, err - } - checkNonExist := func() bool { - return tester.getBlock(hashes[0]) == nil - } - if light { - checkNonExist = func() bool { - return tester.getHeader(hashes[0]) == nil - } - } - // Announce the same block many times until it's fetched (wait for any pending ops) - for checkNonExist() { - tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher) - time.Sleep(time.Millisecond) - } - time.Sleep(delay) - - // Check that all blocks were imported and none fetched twice - if c := counter.Load(); c != 1 { - t.Fatalf("retrieval count mismatch: have %v, want %v", c, 1) - } - verifyChainHeight(t, tester, 1) -} - -// Tests that announcements retrieved in a random order are cached and eventually -// imported when all the gaps are filled in. -func TestFullRandomArrivalImport(t *testing.T) { testRandomArrivalImport(t, false) } -func TestLightRandomArrivalImport(t *testing.T) { testRandomArrivalImport(t, true) } - -func testRandomArrivalImport(t *testing.T, light bool) { - // Create a chain of blocks to import, and choose one to delay - targetBlocks := maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - skip := targetBlocks / 2 - - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, skipping one entry - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 1; i >= 0; i-- { - if i != skip { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - time.Sleep(time.Millisecond) - } - } - // Finally announce the skipped entry and check full import - tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - verifyImportCount(t, imported, len(hashes)-1) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that direct block enqueues (due to block propagation vs. hash announce) -// are correctly schedule, filling and import queue gaps. -func TestQueueGapFill(t *testing.T) { - // Create a chain of blocks to import, and choose one to not announce at all - targetBlocks := maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - skip := targetBlocks / 2 - - tester := newTester(false) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, skipping one entry - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - - for i := len(hashes) - 1; i >= 0; i-- { - if i != skip { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - time.Sleep(time.Millisecond) - } - } - // Fill the missing block directly as if propagated - tester.fetcher.Enqueue("valid", blocks[hashes[skip]]) - verifyImportCount(t, imported, len(hashes)-1) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that blocks arriving from various sources (multiple propagations, hash -// announces, etc) do not get scheduled for import multiple times. -func TestImportDeduplication(t *testing.T) { - // Create two blocks to import (one for duplication, the other for stalling) - hashes, blocks := makeChain(2, 0, genesis) - - // Create the tester and wrap the importer with a counter - tester := newTester(false) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - var counter atomic.Uint32 - tester.fetcher.insertChain = func(blocks types.Blocks) (int, error) { - counter.Add(uint32(len(blocks))) - return tester.insertChain(blocks) - } - // Instrument the fetching and imported events - fetching := make(chan []common.Hash) - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - - // Announce the duplicating block, wait for retrieval, and also propagate directly - tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - <-fetching - - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - - // Fill the missing block directly as if propagated, and check import uniqueness - tester.fetcher.Enqueue("valid", blocks[hashes[1]]) - verifyImportCount(t, imported, 2) - - if c := counter.Load(); c != 2 { - t.Fatalf("import invocation count mismatch: have %v, want %v", c, 2) - } -} - -// Tests that blocks with numbers much lower or higher than out current head get -// discarded to prevent wasting resources on useless blocks from faulty peers. -func TestDistantPropagationDiscarding(t *testing.T) { - // Create a long chain to import and define the discard boundaries - hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) - head := hashes[len(hashes)/2] - - low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1 - - // Create a tester and simulate a head block being the middle of the above chain - tester := newTester(false) - - tester.lock.Lock() - tester.hashes = []common.Hash{head} - tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} - tester.lock.Unlock() - - // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Enqueue("lower", blocks[hashes[low]]) - time.Sleep(10 * time.Millisecond) - if !tester.fetcher.queue.Empty() { - t.Fatalf("fetcher queued stale block") - } - // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Enqueue("higher", blocks[hashes[high]]) - time.Sleep(10 * time.Millisecond) - if !tester.fetcher.queue.Empty() { - t.Fatalf("fetcher queued future block") - } -} - -// Tests that announcements with numbers much lower or higher than out current -// head get discarded to prevent wasting resources on useless blocks from faulty -// peers. -func TestFullDistantAnnouncementDiscarding(t *testing.T) { testDistantAnnouncementDiscarding(t, false) } -func TestLightDistantAnnouncementDiscarding(t *testing.T) { testDistantAnnouncementDiscarding(t, true) } - -func testDistantAnnouncementDiscarding(t *testing.T, light bool) { - // Create a long chain to import and define the discard boundaries - hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) - head := hashes[len(hashes)/2] - - low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1 - - // Create a tester and simulate a head block being the middle of the above chain - tester := newTester(light) - - tester.lock.Lock() - tester.hashes = []common.Hash{head} - tester.headers = map[common.Hash]*types.Header{head: blocks[head].Header()} - tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} - tester.lock.Unlock() - - headerFetcher := tester.makeHeaderFetcher("lower", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("lower", blocks, 0) - - fetching := make(chan struct{}, 2) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- struct{}{} } - - // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-time.After(50 * time.Millisecond): - case <-fetching: - t.Fatalf("fetcher requested stale header") - } - // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-time.After(50 * time.Millisecond): - case <-fetching: - t.Fatalf("fetcher requested future header") - } -} - -// Tests that peers announcing blocks with invalid numbers (i.e. not matching -// the headers provided afterwards) get dropped as malicious. -func TestFullInvalidNumberAnnouncement(t *testing.T) { testInvalidNumberAnnouncement(t, false) } -func TestLightInvalidNumberAnnouncement(t *testing.T) { testInvalidNumberAnnouncement(t, true) } - -func testInvalidNumberAnnouncement(t *testing.T, light bool) { - // Create a single block to import and check numbers against - hashes, blocks := makeChain(1, 0, genesis) - - tester := newTester(light) - badHeaderFetcher := tester.makeHeaderFetcher("bad", blocks, -gatherSlack) - badBodyFetcher := tester.makeBodyFetcher("bad", blocks, 0) - - imported := make(chan interface{}) - announced := make(chan interface{}, 2) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - // Announce a block with a bad number, check for immediate drop - tester.fetcher.announceChangeHook = func(hash common.Hash, b bool) { - announced <- nil - } - tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), badHeaderFetcher, badBodyFetcher) - verifyAnnounce := func() { - for i := 0; i < 2; i++ { - select { - case <-announced: - continue - case <-time.After(1 * time.Second): - t.Fatal("announce timeout") - return - } - } - } - verifyAnnounce() - verifyImportEvent(t, imported, false) - tester.lock.RLock() - dropped := tester.drops["bad"] - tester.lock.RUnlock() - - if !dropped { - t.Fatalf("peer with invalid numbered announcement not dropped") - } - goodHeaderFetcher := tester.makeHeaderFetcher("good", blocks, -gatherSlack) - goodBodyFetcher := tester.makeBodyFetcher("good", blocks, 0) - // Make sure a good announcement passes without a drop - tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), goodHeaderFetcher, goodBodyFetcher) - verifyAnnounce() - verifyImportEvent(t, imported, true) - - tester.lock.RLock() - dropped = tester.drops["good"] - tester.lock.RUnlock() - - if dropped { - t.Fatalf("peer with valid numbered announcement dropped") - } - verifyImportDone(t, imported) -} - -// Tests that if a block is empty (i.e. header only), no body request should be -// made, and instead the header should be assembled into a whole block in itself. -func TestEmptyBlockShortCircuit(t *testing.T) { - // Create a chain of blocks to import - hashes, blocks := makeChain(32, 0, genesis) - - tester := newTester(false) - defer tester.fetcher.Stop() - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Add a monitoring hook for all internal events - fetching := make(chan []common.Hash) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } - - completing := make(chan []common.Hash) - tester.fetcher.completingHook = func(hashes []common.Hash) { completing <- hashes } - - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - // Iteratively announce blocks until all are imported - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - - // All announces should fetch the header - verifyFetchingEvent(t, fetching, true) - - // Only blocks with data contents should request bodies - verifyCompletingEvent(t, completing, len(blocks[hashes[i]].Transactions()) > 0 || len(blocks[hashes[i]].Uncles()) > 0) - - // Irrelevant of the construct, import should succeed - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} - -// Tests that a peer is unable to use unbounded memory with sending infinite -// block announcements to a node, but that even in the face of such an attack, -// the fetcher remains operational. -func TestHashMemoryExhaustionAttack(t *testing.T) { - // Create a tester with instrumented import hooks - tester := newTester(false) - - imported, announces := make(chan interface{}), atomic.Int32{} - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - tester.fetcher.announceChangeHook = func(hash common.Hash, added bool) { - if added { - announces.Add(1) - } else { - announces.Add(-1) - } - } - // Create a valid chain and an infinite junk chain - targetBlocks := hashLimit + 2*maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - validHeaderFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - validBodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - attack, _ := makeChain(targetBlocks, 0, unknownBlock) - attackerHeaderFetcher := tester.makeHeaderFetcher("attacker", nil, -gatherSlack) - attackerBodyFetcher := tester.makeBodyFetcher("attacker", nil, 0) - - // Feed the tester a huge hashset from the attacker, and a limited from the valid peer - for i := 0; i < len(attack); i++ { - if i < maxQueueDist { - tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), validHeaderFetcher, validBodyFetcher) - } - tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), attackerHeaderFetcher, attackerBodyFetcher) - } - if count := announces.Load(); count != hashLimit+maxQueueDist { - t.Fatalf("queued announce count mismatch: have %d, want %d", count, hashLimit+maxQueueDist) - } - // Wait for fetches to complete - verifyImportCount(t, imported, maxQueueDist) - - // Feed the remaining valid hashes to ensure DOS protection state remains clean - for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), validHeaderFetcher, validBodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} - -// Tests that blocks sent to the fetcher (either through propagation or via hash -// announces and retrievals) don't pile up indefinitely, exhausting available -// system memory. -func TestBlockMemoryExhaustionAttack(t *testing.T) { - // Create a tester with instrumented import hooks - tester := newTester(false) - - imported, enqueued := make(chan interface{}), atomic.Int32{} - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - tester.fetcher.queueChangeHook = func(hash common.Hash, added bool) { - if added { - enqueued.Add(1) - } else { - enqueued.Add(-1) - } - } - // Create a valid chain and a batch of dangling (but in range) blocks - targetBlocks := hashLimit + 2*maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - attack := make(map[common.Hash]*types.Block) - for i := byte(0); len(attack) < blockLimit+2*maxQueueDist; i++ { - hashes, blocks := makeChain(maxQueueDist-1, i, unknownBlock) - for _, hash := range hashes[:maxQueueDist-2] { - attack[hash] = blocks[hash] - } - } - // Try to feed all the attacker blocks make sure only a limited batch is accepted - for _, block := range attack { - tester.fetcher.Enqueue("attacker", block) - } - time.Sleep(200 * time.Millisecond) - if queued := enqueued.Load(); queued != blockLimit { - t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit) - } - // Queue up a batch of valid blocks, and check that a new peer is allowed to do so - for i := 0; i < maxQueueDist-1; i++ { - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-3-i]]) - } - time.Sleep(100 * time.Millisecond) - if queued := enqueued.Load(); queued != blockLimit+maxQueueDist-1 { - t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit+maxQueueDist-1) - } - // Insert the missing piece (and sanity check the import) - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2]]) - verifyImportCount(t, imported, maxQueueDist) - - // Insert the remaining blocks in chunks to ensure clean DOS protection - for i := maxQueueDist; i < len(hashes)-1; i++ { - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2-i]]) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index ea7892d8d84e..18c5ff007a9f 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -107,6 +107,8 @@ var ( txFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/fetching/hashes", nil) ) +var errTerminated = errors.New("terminated") + // txAnnounce is the notification of the availability of a batch // of new transactions in the network. type txAnnounce struct { @@ -783,7 +785,7 @@ func (f *TxFetcher) loop() { // rescheduleWait iterates over all the transactions currently in the waitlist // and schedules the movement into the fetcher for the earliest. // -// The method has a granularity of 'gatherSlack', since there's not much point in +// The method has a granularity of 'txGatherSlack', since there's not much point in // spinning over all the transactions just to maybe find one that should trigger // a few ms earlier. func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { @@ -796,7 +798,7 @@ func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { for _, instance := range f.waittime { if earliest > instance { earliest = instance - if txArriveTimeout-time.Duration(now-earliest) < gatherSlack { + if txArriveTimeout-time.Duration(now-earliest) < txGatherSlack { break } } @@ -809,7 +811,7 @@ func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { // rescheduleTimeout iterates over all the transactions currently in flight and // schedules a cleanup run when the first would trigger. // -// The method has a granularity of 'gatherSlack', since there's not much point in +// The method has a granularity of 'txGatherSlack', since there's not much point in // spinning over all the transactions just to maybe find one that should trigger // a few ms earlier. // @@ -834,7 +836,7 @@ func (f *TxFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{} } if earliest > req.time { earliest = req.time - if txFetchTimeout-time.Duration(now-earliest) < gatherSlack { + if txFetchTimeout-time.Duration(now-earliest) < txGatherSlack { break } } diff --git a/eth/filters/api.go b/eth/filters/api.go index 8cf701ec5713..59103ac03ca6 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -179,8 +179,6 @@ func (api *FilterAPI) NewPendingTransactions(ctx context.Context, fullTx *bool) } case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() @@ -241,8 +239,6 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { notifier.Notify(rpcSub.ID, h) case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() @@ -278,8 +274,6 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc } case <-rpcSub.Err(): // client send an unsubscribe request return - case <-notifier.Closed(): // connection dropped - return } } }() diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 83e3284a2b5b..f2b92d5a99da 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -333,7 +333,7 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*typ // pendingLogs returns the logs matching the filter criteria within the pending block. func (f *Filter) pendingLogs() []*types.Log { - block, receipts := f.sys.backend.PendingBlockAndReceipts() + block, receipts, _ := f.sys.backend.Pending() if block == nil || receipts == nil { return nil } diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index f98a1f84ce14..c32b837eb477 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -62,7 +63,7 @@ type Backend interface { GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) + Pending() (*types.Block, types.Receipts, *state.StateDB) CurrentHeader() *types.Header ChainConfig() *params.ChainConfig @@ -70,7 +71,6 @@ type Backend interface { SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription BloomStatus() (uint64, uint64) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) @@ -198,20 +198,18 @@ type EventSystem struct { lastHead *types.Header // Subscriptions - txsSub event.Subscription // Subscription for new transaction event - logsSub event.Subscription // Subscription for new log event - rmLogsSub event.Subscription // Subscription for removed log event - pendingLogsSub event.Subscription // Subscription for pending log event - chainSub event.Subscription // Subscription for new chain event + txsSub event.Subscription // Subscription for new transaction event + logsSub event.Subscription // Subscription for new log event + rmLogsSub event.Subscription // Subscription for removed log event + chainSub event.Subscription // Subscription for new chain event // Channels - install chan *subscription // install filter for event notification - uninstall chan *subscription // remove filter for event notification - txsCh chan core.NewTxsEvent // Channel to receive new transactions event - logsCh chan []*types.Log // Channel to receive new log event - pendingLogsCh chan []*types.Log // Channel to receive new log event - rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event - chainCh chan core.ChainEvent // Channel to receive new chain event + install chan *subscription // install filter for event notification + uninstall chan *subscription // remove filter for event notification + txsCh chan core.NewTxsEvent // Channel to receive new transactions event + logsCh chan []*types.Log // Channel to receive new log event + rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event + chainCh chan core.ChainEvent // Channel to receive new chain event } // NewEventSystem creates a new manager that listens for event on the given mux, @@ -222,16 +220,15 @@ type EventSystem struct { // or by stopping the given mux. func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m := &EventSystem{ - sys: sys, - backend: sys.backend, - lightMode: lightMode, - install: make(chan *subscription), - uninstall: make(chan *subscription), - txsCh: make(chan core.NewTxsEvent, txChanSize), - logsCh: make(chan []*types.Log, logsChanSize), - rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), - pendingLogsCh: make(chan []*types.Log, logsChanSize), - chainCh: make(chan core.ChainEvent, chainEvChanSize), + sys: sys, + backend: sys.backend, + lightMode: lightMode, + install: make(chan *subscription), + uninstall: make(chan *subscription), + txsCh: make(chan core.NewTxsEvent, txChanSize), + logsCh: make(chan []*types.Log, logsChanSize), + rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), + chainCh: make(chan core.ChainEvent, chainEvChanSize), } // Subscribe events @@ -239,10 +236,9 @@ func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m.logsSub = m.backend.SubscribeLogsEvent(m.logsCh) m.rmLogsSub = m.backend.SubscribeRemovedLogsEvent(m.rmLogsCh) m.chainSub = m.backend.SubscribeChainEvent(m.chainCh) - m.pendingLogsSub = m.backend.SubscribePendingLogsEvent(m.pendingLogsCh) // Make sure none of the subscriptions are empty - if m.txsSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil || m.pendingLogsSub == nil { + if m.txsSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil { log.Crit("Subscribe for event system failed") } @@ -434,12 +430,12 @@ func (es *EventSystem) handleLogs(filters filterIndex, ev []*types.Log) { } } -func (es *EventSystem) handlePendingLogs(filters filterIndex, ev []*types.Log) { - if len(ev) == 0 { +func (es *EventSystem) handlePendingLogs(filters filterIndex, logs []*types.Log) { + if len(logs) == 0 { return } for _, f := range filters[PendingLogsSubscription] { - matchedLogs := filterLogs(ev, nil, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics) + matchedLogs := filterLogs(logs, nil, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics) if len(matchedLogs) > 0 { f.logs <- matchedLogs } @@ -550,7 +546,6 @@ func (es *EventSystem) eventLoop() { es.txsSub.Unsubscribe() es.logsSub.Unsubscribe() es.rmLogsSub.Unsubscribe() - es.pendingLogsSub.Unsubscribe() es.chainSub.Unsubscribe() }() @@ -567,10 +562,29 @@ func (es *EventSystem) eventLoop() { es.handleLogs(index, ev) case ev := <-es.rmLogsCh: es.handleLogs(index, ev.Logs) - case ev := <-es.pendingLogsCh: - es.handlePendingLogs(index, ev) case ev := <-es.chainCh: es.handleChainEvent(index, ev) + // If we have no pending log subscription, + // we don't need to collect any pending logs. + if len(index[PendingLogsSubscription]) == 0 { + continue + } + + // Pull the pending logs if there is a new chain head. + pendingBlock, pendingReceipts, _ := es.backend.Pending() + if pendingBlock == nil || pendingReceipts == nil { + continue + } + if pendingBlock.ParentHash() != ev.Block.Hash() { + continue + } + var logs []*types.Log + for _, receipt := range pendingReceipts { + if len(receipt.Logs) > 0 { + logs = append(logs, receipt.Logs...) + } + } + es.handlePendingLogs(index, logs) case f := <-es.install: if f.typ == MinedAndPendingLogsSubscription { diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 99c012cc84f4..6238c9773522 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -48,7 +49,6 @@ type testBackend struct { txFeed event.Feed logsFeed event.Feed rmLogsFeed event.Feed - pendingLogsFeed event.Feed chainFeed event.Feed pendingBlock *types.Block pendingReceipts types.Receipts @@ -125,8 +125,8 @@ func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash, number uint return logs, nil } -func (b *testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { - return b.pendingBlock, b.pendingReceipts +func (b *testBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { + return b.pendingBlock, b.pendingReceipts, nil } func (b *testBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { @@ -141,10 +141,6 @@ func (b *testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript return b.logsFeed.Subscribe(ch) } -func (b *testBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - return b.pendingLogsFeed.Subscribe(ch) -} - func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.chainFeed.Subscribe(ch) } @@ -180,6 +176,20 @@ func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.Matc }() } +func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) { + b.pendingBlock = block + b.pendingReceipts = receipts +} + +func (b *testBackend) notifyPending(logs []*types.Log) { + genesis := &core.Genesis{ + Config: params.TestChainConfig, + } + _, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 2, func(i int, b *core.BlockGen) {}) + b.setPending(blocks[1], []*types.Receipt{{Logs: logs}}) + b.chainFeed.Send(core.ChainEvent{Block: blocks[0]}) +} + func newTestFilterSystem(t testing.TB, db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { backend := &testBackend{db: db} sys := NewFilterSystem(backend, cfg) @@ -203,7 +213,7 @@ func TestBlockSubscription(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), } _, chain, _ = core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 10, func(i int, gen *core.BlockGen) {}) - chainEvents = []core.ChainEvent{} + chainEvents []core.ChainEvent ) for _, blk := range chain { @@ -386,7 +396,7 @@ func TestLogFilterCreation(t *testing.T) { {FilterCriteria{FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(100)}, false}, // from block "higher" than to block {FilterCriteria{FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, false}, - // topics more then 4 + // topics more than 4 {FilterCriteria{Topics: [][]common.Hash{{}, {}, {}, {}, {}}}, false}, } ) @@ -546,9 +556,9 @@ func TestLogFilter(t *testing.T) { if nsend := backend.logsFeed.Send(allLogs); nsend == 0 { t.Fatal("Logs event not delivered") } - if nsend := backend.pendingLogsFeed.Send(allLogs); nsend == 0 { - t.Fatal("Pending logs event not delivered") - } + + // set pending logs + backend.notifyPending(allLogs) for i, tt := range testCases { var fetched []*types.Log @@ -754,10 +764,12 @@ func TestPendingLogsSubscription(t *testing.T) { }() } - // raise events - for _, ev := range allLogs { - backend.pendingLogsFeed.Send(ev) + // set pending logs + var flattenLogs []*types.Log + for _, logs := range allLogs { + flattenLogs = append(flattenLogs, logs...) } + backend.notifyPending(flattenLogs) for i := range testCases { err := <-testCases[i].err diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 659ca5ce197d..48aaa584dbb1 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -109,8 +109,8 @@ func BenchmarkFilters(b *testing.B) { func TestFilters(t *testing.T) { var ( - db = rawdb.NewMemoryDatabase() - _, sys = newTestFilterSystem(t, db, Config{}) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) // Sender account key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) @@ -277,8 +277,7 @@ func TestFilters(t *testing.T) { }), signer, key1) gen.AddTx(tx) }) - sys.backend.(*testBackend).pendingBlock = pchain[0] - sys.backend.(*testBackend).pendingReceipts = preceipts[0] + backend.setPending(pchain[0], preceipts[0]) for i, tc := range []struct { f *Filter diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index d657eb6d996b..8ab57294b7e8 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -160,7 +160,7 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, reqEnd rpc.BlockNum ) switch reqEnd { case rpc.PendingBlockNumber: - if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { + if pendingBlock, pendingReceipts, _ = oracle.backend.Pending(); pendingBlock != nil { resolved = pendingBlock.Header() } else { // Pending block not supported by backend, process only until latest block. diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index b71964981145..3fa70e41a094 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" @@ -54,7 +55,7 @@ type OracleBackend interface { HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) + Pending() (*types.Block, types.Receipts, *state.StateDB) ChainConfig() *params.ChainConfig SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription } diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 79217502f799..1d2e02cde6e1 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -97,12 +98,13 @@ func (b *testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types. return b.chain.GetReceiptsByHash(hash), nil } -func (b *testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { +func (b *testBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { if b.pending { block := b.chain.GetBlockByNumber(testHead + 1) - return block, b.chain.GetReceiptsByHash(block.Hash()) + state, _ := b.chain.StateAt(block.Root()) + return block, b.chain.GetReceiptsByHash(block.Hash()), state } - return nil, nil + return nil, nil, nil } func (b *testBackend) ChainConfig() *params.ChainConfig { diff --git a/eth/handler.go b/eth/handler.go index b2fef62ea3c4..0d27e061c49b 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -25,13 +25,12 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -41,8 +40,9 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/triedb/pathdb" - "github.com/holiman/uint256" + "golang.org/x/crypto/sha3" ) const ( @@ -74,7 +74,7 @@ type txPool interface { // Pending should return pending transactions. // The slice should be modifiable by the caller. - Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction + Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions @@ -85,10 +85,10 @@ type txPool interface { // handlerConfig is the collection of initialization parameters to create a full // node network handler. type handlerConfig struct { + NodeID enode.ID // P2P node ID used for tx propagation topology Database ethdb.Database // Database for direct sync insertions Chain *core.BlockChain // Blockchain to serve data from TxPool txPool // Transaction pool to propagate from - Merger *consensus.Merger // The manager for eth1/2 transition Network uint64 // Network identifier to advertise Sync downloader.SyncMode // Whether to snap or full sync BloomCache uint64 // Megabytes to alloc for snap sync bloom @@ -97,6 +97,7 @@ type handlerConfig struct { } type handler struct { + nodeID enode.ID networkID uint64 forkFilter forkid.Filter // Fork ID filter, constant across the lifetime of the node @@ -108,24 +109,20 @@ type handler struct { chain *core.BlockChain maxPeers int - downloader *downloader.Downloader - blockFetcher *fetcher.BlockFetcher - txFetcher *fetcher.TxFetcher - peers *peerSet - merger *consensus.Merger + downloader *downloader.Downloader + txFetcher *fetcher.TxFetcher + peers *peerSet - eventMux *event.TypeMux - txsCh chan core.NewTxsEvent - txsSub event.Subscription - minedBlockSub *event.TypeMuxSubscription + eventMux *event.TypeMux + txsCh chan core.NewTxsEvent + txsSub event.Subscription requiredBlocks map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop quitSync chan struct{} - chainSync *chainSyncer - wg sync.WaitGroup + wg sync.WaitGroup handlerStartCh chan struct{} handlerDoneCh chan struct{} @@ -138,6 +135,7 @@ func newHandler(config *handlerConfig) (*handler, error) { config.EventMux = new(event.TypeMux) // Nicety initialization for tests } h := &handler{ + nodeID: config.NodeID, networkID: config.Network, forkFilter: forkid.NewFilter(config.Chain), eventMux: config.EventMux, @@ -145,7 +143,6 @@ func newHandler(config *handlerConfig) (*handler, error) { txpool: config.TxPool, chain: config.Chain, peers: newPeerSet(), - merger: config.Merger, requiredBlocks: config.RequiredBlocks, quitSync: make(chan struct{}), handlerDoneCh: make(chan struct{}), @@ -185,92 +182,6 @@ func newHandler(config *handlerConfig) (*handler, error) { } // Construct the downloader (long sync) h.downloader = downloader.New(config.Database, h.eventMux, h.chain, nil, h.removePeer, h.enableSyncedFeatures) - if ttd := h.chain.Config().TerminalTotalDifficulty; ttd != nil { - if h.chain.Config().TerminalTotalDifficultyPassed { - log.Info("Chain post-merge, sync via beacon client") - } else { - head := h.chain.CurrentBlock() - if td := h.chain.GetTd(head.Hash(), head.Number.Uint64()); td.Cmp(ttd) >= 0 { - log.Info("Chain post-TTD, sync via beacon client") - } else { - log.Warn("Chain pre-merge, sync via PoW (ensure beacon client is ready)") - } - } - } else if h.chain.Config().TerminalTotalDifficultyPassed { - log.Error("Chain configured post-merge, but without TTD. Are you debugging sync?") - } - // Construct the fetcher (short sync) - validator := func(header *types.Header) error { - // All the block fetcher activities should be disabled - // after the transition. Print the warning log. - if h.merger.PoSFinalized() { - log.Warn("Unexpected validation activity", "hash", header.Hash(), "number", header.Number) - return errors.New("unexpected behavior after transition") - } - // Reject all the PoS style headers in the first place. No matter - // the chain has finished the transition or not, the PoS headers - // should only come from the trusted consensus layer instead of - // p2p network. - if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { - if beacon.IsPoSHeader(header) { - return errors.New("unexpected post-merge header") - } - } - return h.chain.Engine().VerifyHeader(h.chain, header) - } - heighter := func() uint64 { - return h.chain.CurrentBlock().Number.Uint64() - } - inserter := func(blocks types.Blocks) (int, error) { - // All the block fetcher activities should be disabled - // after the transition. Print the warning log. - if h.merger.PoSFinalized() { - var ctx []interface{} - ctx = append(ctx, "blocks", len(blocks)) - if len(blocks) > 0 { - ctx = append(ctx, "firsthash", blocks[0].Hash()) - ctx = append(ctx, "firstnumber", blocks[0].Number()) - ctx = append(ctx, "lasthash", blocks[len(blocks)-1].Hash()) - ctx = append(ctx, "lastnumber", blocks[len(blocks)-1].Number()) - } - log.Warn("Unexpected insertion activity", ctx...) - return 0, errors.New("unexpected behavior after transition") - } - // If snap sync is running, deny importing weird blocks. This is a problematic - // clause when starting up a new network, because snap-syncing miners might not - // accept each others' blocks until a restart. Unfortunately we haven't figured - // out a way yet where nodes can decide unilaterally whether the network is new - // or not. This should be fixed if we figure out a solution. - if !h.synced.Load() { - log.Warn("Syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) - return 0, nil - } - if h.merger.TDDReached() { - // The blocks from the p2p network is regarded as untrusted - // after the transition. In theory block gossip should be disabled - // entirely whenever the transition is started. But in order to - // handle the transition boundary reorg in the consensus-layer, - // the legacy blocks are still accepted, but only for the terminal - // pow blocks. Spec: https://github.com/ethereum/EIPs/blob/master/EIPS/eip-3675.md#halt-the-importing-of-pow-blocks - for i, block := range blocks { - ptd := h.chain.GetTd(block.ParentHash(), block.NumberU64()-1) - if ptd == nil { - return 0, nil - } - td := new(big.Int).Add(ptd, block.Difficulty()) - if !h.chain.Config().IsTerminalPoWBlock(ptd, td) { - log.Info("Filtered out non-terminal pow block", "number", block.NumberU64(), "hash", block.Hash()) - return 0, nil - } - if err := h.chain.InsertBlockWithoutSetHead(block); err != nil { - return i, err - } - } - return 0, nil - } - return h.chain.InsertChain(blocks) - } - h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock, heighter, nil, inserter, h.removePeer) fetchTx := func(peer string, hashes []common.Hash) error { p := h.peers.peer(peer) @@ -283,7 +194,6 @@ func newHandler(config *handlerConfig) (*handler, error) { return h.txpool.Add(txs, false, false) } h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, addTxs, fetchTx, h.removePeer) - h.chainSync = newChainSyncer(h) return h, nil } @@ -393,8 +303,6 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { return err } } - h.chainSync.handlePeerEvent() - // Propagate existing transactions. new transactions appearing // after this will be sent via broadcasts. h.syncTransactions(peer) @@ -521,14 +429,8 @@ func (h *handler) Start(maxPeers int) { h.txsSub = h.txpool.SubscribeTransactions(h.txsCh, false) go h.txBroadcastLoop() - // broadcast mined blocks - h.wg.Add(1) - h.minedBlockSub = h.eventMux.Subscribe(core.NewMinedBlockEvent{}) - go h.minedBroadcastLoop() - // start sync handlers - h.wg.Add(1) - go h.chainSync.loop() + h.txFetcher.Start() // start peer handler tracker h.wg.Add(1) @@ -536,8 +438,9 @@ func (h *handler) Start(maxPeers int) { } func (h *handler) Stop() { - h.txsSub.Unsubscribe() // quits txBroadcastLoop - h.minedBlockSub.Unsubscribe() // quits blockBroadcastLoop + h.txsSub.Unsubscribe() // quits txBroadcastLoop + h.txFetcher.Stop() + h.downloader.Terminate() // Quit chainSync and txsync64. // After this is done, no new peers will be accepted. @@ -553,50 +456,6 @@ func (h *handler) Stop() { log.Info("Ethereum protocol stopped") } -// BroadcastBlock will either propagate a block to a subset of its peers, or -// will only announce its availability (depending what's requested). -func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { - // Disable the block propagation if the chain has already entered the PoS - // stage. The block propagation is delegated to the consensus layer. - if h.merger.PoSFinalized() { - return - } - // Disable the block propagation if it's the post-merge block. - if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { - if beacon.IsPoSHeader(block.Header()) { - return - } - } - hash := block.Hash() - peers := h.peers.peersWithoutBlock(hash) - - // If propagation is requested, send to a subset of the peer - if propagate { - // Calculate the TD of the block (it's not imported yet, so block.Td is not valid) - var td *big.Int - if parent := h.chain.GetBlock(block.ParentHash(), block.NumberU64()-1); parent != nil { - td = new(big.Int).Add(block.Difficulty(), h.chain.GetTd(block.ParentHash(), block.NumberU64()-1)) - } else { - log.Error("Propagating dangling block", "number", block.Number(), "hash", hash) - return - } - // Send the block to a subset of our peers - transfer := peers[:int(math.Sqrt(float64(len(peers))))] - for _, peer := range transfer { - peer.AsyncSendNewBlock(block, td) - } - log.Trace("Propagated block", "hash", hash, "recipients", len(transfer), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) - return - } - // Otherwise if the block is indeed in out own chain, announce it - if h.chain.HasBlock(hash, block.NumberU64()) { - for _, peer := range peers { - peer.AsyncSendNewBlockHash(block) - } - log.Trace("Announced block", "hash", hash, "recipients", len(peers), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) - } -} - // BroadcastTransactions will propagate a batch of transactions // - To a square root of all peers for non-blob transactions // - And, separately, as announcements to all peers which are not known to @@ -615,25 +474,54 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { annos = make(map[*ethPeer][]common.Hash) // Set peer->hash to announce ) // Broadcast transactions to a batch of peers not knowing about it - for _, tx := range txs { - peers := h.peers.peersWithoutTransaction(tx.Hash()) + direct := big.NewInt(int64(math.Sqrt(float64(h.peers.len())))) // Approximate number of peers to broadcast to + if direct.BitLen() == 0 { + direct = big.NewInt(1) + } + total := new(big.Int).Exp(direct, big.NewInt(2), nil) // Stabilise total peer count a bit based on sqrt peers - var numDirect int + var ( + signer = types.LatestSignerForChainID(h.chain.Config().ChainID) // Don't care about chain status, we just need *a* sender + hasher = sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash = make([]byte, 32) + ) + for _, tx := range txs { + var maybeDirect bool switch { case tx.Type() == types.BlobTxType: blobTxs++ case tx.Size() > txMaxBroadcastSize: largeTxs++ default: - numDirect = int(math.Sqrt(float64(len(peers)))) + maybeDirect = true } - // Send the tx unconditionally to a subset of our peers - for _, peer := range peers[:numDirect] { - txset[peer] = append(txset[peer], tx.Hash()) - } - // For the remaining peers, send announcement only - for _, peer := range peers[numDirect:] { - annos[peer] = append(annos[peer], tx.Hash()) + // Send the transaction (if it's small enough) directly to a subset of + // the peers that have not received it yet, ensuring that the flow of + // transactions is grouped by account to (try and) avoid nonce gaps. + // + // To do this, we hash the local enode IW with together with a peer's + // enode ID together with the transaction sender and broadcast if + // `sha(self, peer, sender) mod peers < sqrt(peers)`. + for _, peer := range h.peers.peersWithoutTransaction(tx.Hash()) { + var broadcast bool + if maybeDirect { + hasher.Reset() + hasher.Write(h.nodeID.Bytes()) + hasher.Write(peer.Node().ID().Bytes()) + + from, _ := types.Sender(signer, tx) // Ignore error, we only use the addr as a propagation target splitter + hasher.Write(from.Bytes()) + + hasher.Read(hash) + if new(big.Int).Mod(new(big.Int).SetBytes(hash), total).Cmp(direct) < 0 { + broadcast = true + } + } + if broadcast { + txset[peer] = append(txset[peer], tx.Hash()) + } else { + annos[peer] = append(annos[peer], tx.Hash()) + } } } for peer, hashes := range txset { @@ -650,18 +538,6 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { "bcastpeers", directPeers, "bcastcount", directCount, "annpeers", annPeers, "anncount", annCount) } -// minedBroadcastLoop sends mined blocks to connected peers. -func (h *handler) minedBroadcastLoop() { - defer h.wg.Done() - - for obj := range h.minedBlockSub.Chan() { - if ev, ok := obj.Data.(core.NewMinedBlockEvent); ok { - h.BroadcastBlock(ev.Block, true) // First propagate block to peers - h.BroadcastBlock(ev.Block, false) // Only then announce to the rest - } - } -} - // txBroadcastLoop announces new transactions to connected peers. func (h *handler) txBroadcastLoop() { defer h.wg.Done() diff --git a/eth/handler_eth.go b/eth/handler_eth.go index f1284c10e637..b2cd52a22105 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -19,10 +19,7 @@ package eth import ( "errors" "fmt" - "math/big" - "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" @@ -60,13 +57,6 @@ func (h *ethHandler) AcceptTxs() bool { func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Consume any broadcasts and announces, forwarding the rest to the downloader switch packet := packet.(type) { - case *eth.NewBlockHashesPacket: - hashes, numbers := packet.Unpack() - return h.handleBlockAnnounces(peer, hashes, numbers) - - case *eth.NewBlockPacket: - return h.handleBlockBroadcast(peer, packet.Block, packet.TD) - case *eth.NewPooledTransactionHashesPacket: return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes) @@ -85,55 +75,3 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { return fmt.Errorf("unexpected eth packet type: %T", packet) } } - -// handleBlockAnnounces is invoked from a peer's message handler when it transmits a -// batch of block announcements for the local node to process. -func (h *ethHandler) handleBlockAnnounces(peer *eth.Peer, hashes []common.Hash, numbers []uint64) error { - // Drop all incoming block announces from the p2p network if - // the chain already entered the pos stage and disconnect the - // remote peer. - if h.merger.PoSFinalized() { - return errors.New("disallowed block announcement") - } - // Schedule all the unknown hashes for retrieval - var ( - unknownHashes = make([]common.Hash, 0, len(hashes)) - unknownNumbers = make([]uint64, 0, len(numbers)) - ) - for i := 0; i < len(hashes); i++ { - if !h.chain.HasBlock(hashes[i], numbers[i]) { - unknownHashes = append(unknownHashes, hashes[i]) - unknownNumbers = append(unknownNumbers, numbers[i]) - } - } - for i := 0; i < len(unknownHashes); i++ { - h.blockFetcher.Notify(peer.ID(), unknownHashes[i], unknownNumbers[i], time.Now(), peer.RequestOneHeader, peer.RequestBodies) - } - return nil -} - -// handleBlockBroadcast is invoked from a peer's message handler when it transmits a -// block broadcast for the local node to process. -func (h *ethHandler) handleBlockBroadcast(peer *eth.Peer, block *types.Block, td *big.Int) error { - // Drop all incoming block announces from the p2p network if - // the chain already entered the pos stage and disconnect the - // remote peer. - if h.merger.PoSFinalized() { - return errors.New("disallowed block broadcast") - } - // Schedule the block for import - h.blockFetcher.Enqueue(peer.ID(), block) - - // Assuming the block is importable by the peer, but possibly not yet done so, - // calculate the head hash and TD that the peer truly must have. - var ( - trueHead = block.ParentHash() - trueTD = new(big.Int).Sub(td, block.Difficulty()) - ) - // Update the peer's total difficulty if better than the previous - if _, td := peer.Head(); trueTD.Cmp(td) > 0 { - peer.SetHead(trueHead, trueTD) - h.chainSync.handlePeerEvent() - } - return nil -} diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 579ca3c09735..297a6bf154ed 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" @@ -109,7 +108,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { Database: dbNoFork, Chain: chainNoFork, TxPool: newTestTxPool(), - Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FullSync, BloomCache: 1, @@ -118,7 +116,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { Database: dbProFork, Chain: chainProFork, TxPool: newTestTxPool(), - Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.FullSync, BloomCache: 1, @@ -441,159 +438,3 @@ func testTransactionPropagation(t *testing.T, protocol uint) { } } } - -// Tests that blocks are broadcast to a sqrt number of peers only. -func TestBroadcastBlock1Peer(t *testing.T) { testBroadcastBlock(t, 1, 1) } -func TestBroadcastBlock2Peers(t *testing.T) { testBroadcastBlock(t, 2, 1) } -func TestBroadcastBlock3Peers(t *testing.T) { testBroadcastBlock(t, 3, 1) } -func TestBroadcastBlock4Peers(t *testing.T) { testBroadcastBlock(t, 4, 2) } -func TestBroadcastBlock5Peers(t *testing.T) { testBroadcastBlock(t, 5, 2) } -func TestBroadcastBlock8Peers(t *testing.T) { testBroadcastBlock(t, 9, 3) } -func TestBroadcastBlock12Peers(t *testing.T) { testBroadcastBlock(t, 12, 3) } -func TestBroadcastBlock16Peers(t *testing.T) { testBroadcastBlock(t, 16, 4) } -func TestBroadcastBloc26Peers(t *testing.T) { testBroadcastBlock(t, 26, 5) } -func TestBroadcastBlock100Peers(t *testing.T) { testBroadcastBlock(t, 100, 10) } - -func testBroadcastBlock(t *testing.T, peers, bcasts int) { - t.Parallel() - - // Create a source handler to broadcast blocks from and a number of sinks - // to receive them. - source := newTestHandlerWithBlocks(1) - defer source.close() - - sinks := make([]*testEthHandler, peers) - for i := 0; i < len(sinks); i++ { - sinks[i] = new(testEthHandler) - } - // Interconnect all the sink handlers with the source handler - var ( - genesis = source.chain.Genesis() - td = source.chain.GetTd(genesis.Hash(), genesis.NumberU64()) - ) - for i, sink := range sinks { - sink := sink // Closure for gorotuine below - - sourcePipe, sinkPipe := p2p.MsgPipe() - defer sourcePipe.Close() - defer sinkPipe.Close() - - sourcePeer := eth.NewPeer(eth.ETH68, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH68, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) - defer sourcePeer.Close() - defer sinkPeer.Close() - - go source.handler.runEthPeer(sourcePeer, func(peer *eth.Peer) error { - return eth.Handle((*ethHandler)(source.handler), peer) - }) - if err := sinkPeer.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain)); err != nil { - t.Fatalf("failed to run protocol handshake") - } - go eth.Handle(sink, sinkPeer) - } - // Subscribe to all the transaction pools - blockChs := make([]chan *types.Block, len(sinks)) - for i := 0; i < len(sinks); i++ { - blockChs[i] = make(chan *types.Block, 1) - defer close(blockChs[i]) - - sub := sinks[i].blockBroadcasts.Subscribe(blockChs[i]) - defer sub.Unsubscribe() - } - // Initiate a block propagation across the peers - time.Sleep(100 * time.Millisecond) - header := source.chain.CurrentBlock() - source.handler.BroadcastBlock(source.chain.GetBlock(header.Hash(), header.Number.Uint64()), true) - - // Iterate through all the sinks and ensure the correct number got the block - done := make(chan struct{}, peers) - for _, ch := range blockChs { - ch := ch - go func() { - <-ch - done <- struct{}{} - }() - } - var received int - for { - select { - case <-done: - received++ - - case <-time.After(100 * time.Millisecond): - if received != bcasts { - t.Errorf("broadcast count mismatch: have %d, want %d", received, bcasts) - } - return - } - } -} - -// Tests that a propagated malformed block (uncles or transactions don't match -// with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock68(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH68) } - -func testBroadcastMalformedBlock(t *testing.T, protocol uint) { - t.Parallel() - - // Create a source handler to broadcast blocks from and a number of sinks - // to receive them. - source := newTestHandlerWithBlocks(1) - defer source.close() - - // Create a source handler to send messages through and a sink peer to receive them - p2pSrc, p2pSink := p2p.MsgPipe() - defer p2pSrc.Close() - defer p2pSink.Close() - - src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, source.txpool) - sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, source.txpool) - defer src.Close() - defer sink.Close() - - go source.handler.runEthPeer(src, func(peer *eth.Peer) error { - return eth.Handle((*ethHandler)(source.handler), peer) - }) - // Run the handshake locally to avoid spinning up a sink handler - var ( - genesis = source.chain.Genesis() - td = source.chain.GetTd(genesis.Hash(), genesis.NumberU64()) - ) - if err := sink.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain)); err != nil { - t.Fatalf("failed to run protocol handshake") - } - // After the handshake completes, the source handler should stream the sink - // the blocks, subscribe to inbound network events - backend := new(testEthHandler) - - blocks := make(chan *types.Block, 1) - sub := backend.blockBroadcasts.Subscribe(blocks) - defer sub.Unsubscribe() - - go eth.Handle(backend, sink) - - // Create various combinations of malformed blocks - head := source.chain.CurrentBlock() - block := source.chain.GetBlock(head.Hash(), head.Number.Uint64()) - - malformedUncles := head - malformedUncles.UncleHash[0]++ - malformedTransactions := head - malformedTransactions.TxHash[0]++ - malformedEverything := head - malformedEverything.UncleHash[0]++ - malformedEverything.TxHash[0]++ - - // Try to broadcast all malformations and ensure they all get discarded - for _, header := range []*types.Header{malformedUncles, malformedTransactions, malformedEverything} { - block := types.NewBlockWithHeader(header).WithBody(block.Transactions(), block.Uncles()) - if err := src.SendNewBlock(block, big.NewInt(131136)); err != nil { - t.Fatalf("failed to broadcast block: %v", err) - } - select { - case <-blocks: - t.Fatalf("malformed block forwarded") - case <-time.After(100 * time.Millisecond): - } - } -} diff --git a/eth/handler_test.go b/eth/handler_test.go index 55f5c4486f16..bcc8ea30e415 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -22,7 +22,6 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -93,7 +92,7 @@ func (p *testTxPool) Add(txs []*types.Transaction, local bool, sync bool) []erro } // Pending returns all the transactions known to the pool -func (p *testTxPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction { +func (p *testTxPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { p.lock.RLock() defer p.lock.RUnlock() @@ -112,8 +111,8 @@ func (p *testTxPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) @@ -164,7 +163,6 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Database: db, Chain: chain, TxPool: txpool, - Merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), Network: 1, Sync: downloader.SnapSync, BloomCache: 1, diff --git a/eth/peerset.go b/eth/peerset.go index c0c11e3e85ee..6b0aff226c1c 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -19,7 +19,6 @@ package eth import ( "errors" "fmt" - "math/big" "sync" "github.com/ethereum/go-ethereum/common" @@ -100,7 +99,7 @@ func (ps *peerSet) registerSnapExtension(peer *snap.Peer) error { return nil } -// waitExtensions blocks until all satellite protocols are connected and tracked +// waitSnapExtension blocks until all satellite protocols are connected and tracked // by the peerset. func (ps *peerSet) waitSnapExtension(peer *eth.Peer) (*snap.Peer, error) { // If the peer does not support a compatible `snap`, don't wait @@ -192,21 +191,6 @@ func (ps *peerSet) peer(id string) *ethPeer { return ps.peers[id] } -// peersWithoutBlock retrieves a list of peers that do not have a given block in -// their set of known hashes so it might be propagated to them. -func (ps *peerSet) peersWithoutBlock(hash common.Hash) []*ethPeer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - list := make([]*ethPeer, 0, len(ps.peers)) - for _, p := range ps.peers { - if !p.KnownBlock(hash) { - list = append(list, p) - } - } - return list -} - // peersWithoutTransaction retrieves a list of peers that do not have a given // transaction in their set of known hashes. func (ps *peerSet) peersWithoutTransaction(hash common.Hash) []*ethPeer { @@ -240,24 +224,6 @@ func (ps *peerSet) snapLen() int { return ps.snapPeers } -// peerWithHighestTD retrieves the known peer with the currently highest total -// difficulty, but below the given PoS switchover threshold. -func (ps *peerSet) peerWithHighestTD() *eth.Peer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - var ( - bestPeer *eth.Peer - bestTd *big.Int - ) - for _, p := range ps.peers { - if _, td := p.Head(); bestPeer == nil || td.Cmp(bestTd) > 0 { - bestPeer, bestTd = p.Peer, td - } - } - return bestPeer -} - // close disconnects all peers. func (ps *peerSet) close() { ps.lock.Lock() diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index ad5395cb8dd9..f0ed1d6bc9a0 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -17,8 +17,6 @@ package eth import ( - "math/big" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" ) @@ -29,37 +27,6 @@ const ( maxTxPacketSize = 100 * 1024 ) -// blockPropagation is a block propagation event, waiting for its turn in the -// broadcast queue. -type blockPropagation struct { - block *types.Block - td *big.Int -} - -// broadcastBlocks is a write loop that multiplexes blocks and block announcements -// to the remote peer. The goal is to have an async writer that does not lock up -// node internals and at the same time rate limits queued data. -func (p *Peer) broadcastBlocks() { - for { - select { - case prop := <-p.queuedBlocks: - if err := p.SendNewBlock(prop.block, prop.td); err != nil { - return - } - p.Log().Trace("Propagated block", "number", prop.block.Number(), "hash", prop.block.Hash(), "td", prop.td) - - case block := <-p.queuedBlockAnns: - if err := p.SendNewBlockHashes([]common.Hash{block.Hash()}, []uint64{block.NumberU64()}); err != nil { - return - } - p.Log().Trace("Announced block", "number", block.Number(), "hash", block.Hash()) - - case <-p.term: - return - } - } -} - // broadcastTransactions is a write loop that schedules transaction broadcasts // to the remote peer. The goal is to have an async writer that does not lock up // node internals and at the same time rate limits queued data. diff --git a/eth/protocols/eth/dispatcher.go b/eth/protocols/eth/dispatcher.go index ae98820cd6ac..146eec3f6085 100644 --- a/eth/protocols/eth/dispatcher.go +++ b/eth/protocols/eth/dispatcher.go @@ -136,7 +136,7 @@ func (p *Peer) dispatchRequest(req *Request) error { } } -// dispatchRequest fulfils a pending request and delivers it to the requested +// dispatchResponse fulfils a pending request and delivers it to the requested // sink. func (p *Peer) dispatchResponse(res *Response, metadata func() interface{}) error { resOp := &response{ diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 0275708a6cd5..96656afb1b20 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -18,6 +18,7 @@ package eth import ( "encoding/json" + "errors" "fmt" "github.com/ethereum/go-ethereum/common" @@ -274,43 +275,11 @@ func ServiceGetReceiptsQuery(chain *core.BlockChain, query GetReceiptsRequest) [ } func handleNewBlockhashes(backend Backend, msg Decoder, peer *Peer) error { - // A batch of new block announcements just arrived - ann := new(NewBlockHashesPacket) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - // Mark the hashes as present at the remote node - for _, block := range *ann { - peer.markBlock(block.Hash) - } - // Deliver them all to the backend for queuing - return backend.Handle(peer, ann) + return errors.New("block announcements disallowed") // We dropped support for non-merge networks } func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { - // Retrieve and decode the propagated block - ann := new(NewBlockPacket) - if err := msg.Decode(ann); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - if err := ann.sanityCheck(); err != nil { - return err - } - if hash := types.CalcUncleHash(ann.Block.Uncles()); hash != ann.Block.UncleHash() { - log.Warn("Propagated block has invalid uncles", "have", hash, "exp", ann.Block.UncleHash()) - return nil // TODO(karalabe): return error eventually, but wait a few releases - } - if hash := types.DeriveSha(ann.Block.Transactions(), trie.NewStackTrie(nil)); hash != ann.Block.TxHash() { - log.Warn("Propagated block has invalid body", "have", hash, "exp", ann.Block.TxHash()) - return nil // TODO(karalabe): return error eventually, but wait a few releases - } - ann.Block.ReceivedAt = msg.Time() - ann.Block.ReceivedFrom = peer - - // Mark the peer as owning the block - peer.markBlock(ann.Block.Hash()) - - return backend.Handle(peer, ann) + return errors.New("block broadcasts disallowed") // We dropped support for non-merge networks } func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index caa5239cf98a..94f28f240f86 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -33,10 +33,6 @@ const ( // before starting to randomly evict them. maxKnownTxs = 32768 - // maxKnownBlocks is the maximum block hashes to keep in the known list - // before starting to randomly evict them. - maxKnownBlocks = 1024 - // maxQueuedTxs is the maximum number of transactions to queue up before dropping // older broadcasts. maxQueuedTxs = 4096 @@ -44,16 +40,6 @@ const ( // maxQueuedTxAnns is the maximum number of transaction announcements to queue up // before dropping older announcements. maxQueuedTxAnns = 4096 - - // maxQueuedBlocks is the maximum number of block propagations to queue up before - // dropping broadcasts. There's not much point in queueing stale blocks, so a few - // that might cover uncles should be enough. - maxQueuedBlocks = 4 - - // maxQueuedBlockAnns is the maximum number of block announcements to queue up before - // dropping broadcasts. Similarly to block propagations, there's no point to queue - // above some healthy uncle limit, so use that. - maxQueuedBlockAnns = 4 ) // max is a helper function which returns the larger of the two given integers. @@ -75,10 +61,6 @@ type Peer struct { head common.Hash // Latest advertised head block hash td *big.Int // Latest advertised head block total difficulty - knownBlocks *knownCache // Set of block hashes known to be known by this peer - queuedBlocks chan *blockPropagation // Queue of blocks to broadcast to the peer - queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer - txpool TxPool // Transaction pool used by the broadcasters for liveness checks knownTxs *knownCache // Set of transaction hashes known to be known by this peer txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests @@ -92,28 +74,24 @@ type Peer struct { lock sync.RWMutex // Mutex protecting the internal fields } -// NewPeer create a wrapper for a network connection and negotiated protocol +// NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Peer { peer := &Peer{ - id: p.ID().String(), - Peer: p, - rw: rw, - version: version, - knownTxs: newKnownCache(maxKnownTxs), - knownBlocks: newKnownCache(maxKnownBlocks), - queuedBlocks: make(chan *blockPropagation, maxQueuedBlocks), - queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns), - txBroadcast: make(chan []common.Hash), - txAnnounce: make(chan []common.Hash), - reqDispatch: make(chan *request), - reqCancel: make(chan *cancel), - resDispatch: make(chan *response), - txpool: txpool, - term: make(chan struct{}), + id: p.ID().String(), + Peer: p, + rw: rw, + version: version, + knownTxs: newKnownCache(maxKnownTxs), + txBroadcast: make(chan []common.Hash), + txAnnounce: make(chan []common.Hash), + reqDispatch: make(chan *request), + reqCancel: make(chan *cancel), + resDispatch: make(chan *response), + txpool: txpool, + term: make(chan struct{}), } // Start up all the broadcasters - go peer.broadcastBlocks() go peer.broadcastTransactions() go peer.announceTransactions() go peer.dispatcher() @@ -156,23 +134,11 @@ func (p *Peer) SetHead(hash common.Hash, td *big.Int) { p.td.Set(td) } -// KnownBlock returns whether peer is known to already have a block. -func (p *Peer) KnownBlock(hash common.Hash) bool { - return p.knownBlocks.Contains(hash) -} - // KnownTransaction returns whether peer is known to already have a transaction. func (p *Peer) KnownTransaction(hash common.Hash) bool { return p.knownTxs.Contains(hash) } -// markBlock marks a block as known for the peer, ensuring that the block will -// never be propagated to this particular peer. -func (p *Peer) markBlock(hash common.Hash) { - // If we reached the memory allowance, drop a previously known block hash - p.knownBlocks.Add(hash) -} - // markTransaction marks a transaction as known for the peer, ensuring that it // will never be propagated to this particular peer. func (p *Peer) markTransaction(hash common.Hash) { @@ -248,55 +214,6 @@ func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs [ }) } -// SendNewBlockHashes announces the availability of a number of blocks through -// a hash notification. -func (p *Peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { - // Mark all the block hashes as known, but ensure we don't overflow our limits - p.knownBlocks.Add(hashes...) - - request := make(NewBlockHashesPacket, len(hashes)) - for i := 0; i < len(hashes); i++ { - request[i].Hash = hashes[i] - request[i].Number = numbers[i] - } - return p2p.Send(p.rw, NewBlockHashesMsg, request) -} - -// AsyncSendNewBlockHash queues the availability of a block for propagation to a -// remote peer. If the peer's broadcast queue is full, the event is silently -// dropped. -func (p *Peer) AsyncSendNewBlockHash(block *types.Block) { - select { - case p.queuedBlockAnns <- block: - // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) - default: - p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash()) - } -} - -// SendNewBlock propagates an entire block to a remote peer. -func (p *Peer) SendNewBlock(block *types.Block, td *big.Int) error { - // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) - return p2p.Send(p.rw, NewBlockMsg, &NewBlockPacket{ - Block: block, - TD: td, - }) -} - -// AsyncSendNewBlock queues an entire block for propagation to a remote peer. If -// the peer's broadcast queue is full, the event is silently dropped. -func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { - select { - case p.queuedBlocks <- &blockPropagation{block: block, td: td}: - // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) - default: - p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash()) - } -} - // ReplyBlockHeadersRLP is the response to GetBlockHeaders. func (p *Peer) ReplyBlockHeadersRLP(id uint64, headers []rlp.RawValue) error { return p2p.Send(p.rw, BlockHeadersMsg, &BlockHeadersRLPPacket{ diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 47e8d97244cb..c5cb2dd1dca4 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -189,19 +189,6 @@ type NewBlockPacket struct { TD *big.Int } -// sanityCheck verifies that the values are reasonable, as a DoS protection -func (request *NewBlockPacket) sanityCheck() error { - if err := request.Block.SanityCheck(); err != nil { - return err - } - //TD at mainnet block #7753254 is 76 bits. If it becomes 100 million times - // larger, it will still fit within 100 bits - if tdlen := request.TD.BitLen(); tdlen > 100 { - return fmt.Errorf("too large block TD: bitlen %d", tdlen) - } - return nil -} - // GetBlockBodiesRequest represents a block body query. type GetBlockBodiesRequest []common.Hash diff --git a/eth/protocols/snap/metrics.go b/eth/protocols/snap/metrics.go index a7d071953f67..a8dc2b582432 100644 --- a/eth/protocols/snap/metrics.go +++ b/eth/protocols/snap/metrics.go @@ -17,7 +17,7 @@ package snap import ( - metrics "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/metrics" ) var ( diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index 3db6e22cbd92..c57931678cec 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -33,7 +33,7 @@ type Peer struct { logger log.Logger // Contextual logger with the peer id injected } -// NewPeer create a wrapper for a network connection and negotiated protocol +// NewPeer creates a wrapper for a network connection and negotiated protocol // version. func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { id := p.ID().String() @@ -46,7 +46,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { } } -// NewFakePeer create a fake snap peer without a backing p2p peer, for testing purposes. +// NewFakePeer creates a fake snap peer without a backing p2p peer, for testing purposes. func NewFakePeer(version uint, id string, rw p2p.MsgReadWriter) *Peer { return &Peer{ id: id, diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index b780868b4e06..cea83aa2bc8e 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1502,7 +1502,7 @@ func getCodeByHash(hash common.Hash) []byte { return nil } -// makeAccountTrieNoStorage spits out a trie, along with the leafs +// makeAccountTrieNoStorage spits out a trie, along with the leaves func makeAccountTrieNoStorage(n int, scheme string) (string, *trie.Trie, []*kv) { var ( db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) @@ -1650,7 +1650,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots return db.Scheme(), accTrie, entries, storageTries, storageEntries } -// makeAccountTrieWithStorage spits out a trie, along with the leafs +// makeAccountTrieWithStorage spits out a trie, along with the leaves func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, boundary bool, uneven bool) (*trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( db = triedb.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) diff --git a/eth/sync.go b/eth/sync.go index fa3a4088043e..61f2b2b376cd 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -17,26 +17,15 @@ package eth import ( - "errors" - "math/big" - "time" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/log" -) - -const ( - forceSyncCycle = 10 * time.Second // Time interval to force syncs, even if few peers are available - defaultMinSyncPeers = 5 // Amount of peers desired to start syncing ) // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { var hashes []common.Hash - for _, batch := range h.txpool.Pending(nil, nil, nil) { + for _, batch := range h.txpool.Pending(txpool.PendingFilter{OnlyPlainTxs: true}) { for _, tx := range batch { hashes = append(hashes, tx.Hash) } @@ -46,206 +35,3 @@ func (h *handler) syncTransactions(p *eth.Peer) { } p.AsyncSendPooledTransactionHashes(hashes) } - -// chainSyncer coordinates blockchain sync components. -type chainSyncer struct { - handler *handler - force *time.Timer - forced bool // true when force timer fired - warned time.Time - peerEventCh chan struct{} - doneCh chan error // non-nil when sync is running -} - -// chainSyncOp is a scheduled sync operation. -type chainSyncOp struct { - mode downloader.SyncMode - peer *eth.Peer - td *big.Int - head common.Hash -} - -// newChainSyncer creates a chainSyncer. -func newChainSyncer(handler *handler) *chainSyncer { - return &chainSyncer{ - handler: handler, - peerEventCh: make(chan struct{}), - } -} - -// handlePeerEvent notifies the syncer about a change in the peer set. -// This is called for new peers and every time a peer announces a new -// chain head. -func (cs *chainSyncer) handlePeerEvent() bool { - select { - case cs.peerEventCh <- struct{}{}: - return true - case <-cs.handler.quitSync: - return false - } -} - -// loop runs in its own goroutine and launches the sync when necessary. -func (cs *chainSyncer) loop() { - defer cs.handler.wg.Done() - - cs.handler.blockFetcher.Start() - cs.handler.txFetcher.Start() - defer cs.handler.blockFetcher.Stop() - defer cs.handler.txFetcher.Stop() - defer cs.handler.downloader.Terminate() - - // The force timer lowers the peer count threshold down to one when it fires. - // This ensures we'll always start sync even if there aren't enough peers. - cs.force = time.NewTimer(forceSyncCycle) - defer cs.force.Stop() - - for { - if op := cs.nextSyncOp(); op != nil { - cs.startSync(op) - } - select { - case <-cs.peerEventCh: - // Peer information changed, recheck. - case err := <-cs.doneCh: - cs.doneCh = nil - cs.force.Reset(forceSyncCycle) - cs.forced = false - - // If we've reached the merge transition but no beacon client is available, or - // it has not yet switched us over, keep warning the user that their infra is - // potentially flaky. - if errors.Is(err, downloader.ErrMergeTransition) && time.Since(cs.warned) > 10*time.Second { - log.Warn("Local chain is post-merge, waiting for beacon client sync switch-over...") - cs.warned = time.Now() - } - case <-cs.force.C: - cs.forced = true - - case <-cs.handler.quitSync: - // Disable all insertion on the blockchain. This needs to happen before - // terminating the downloader because the downloader waits for blockchain - // inserts, and these can take a long time to finish. - cs.handler.chain.StopInsert() - cs.handler.downloader.Terminate() - if cs.doneCh != nil { - <-cs.doneCh - } - return - } - } -} - -// nextSyncOp determines whether sync is required at this time. -func (cs *chainSyncer) nextSyncOp() *chainSyncOp { - if cs.doneCh != nil { - return nil // Sync already running - } - // If a beacon client once took over control, disable the entire legacy sync - // path from here on end. Note, there is a slight "race" between reaching TTD - // and the beacon client taking over. The downloader will enforce that nothing - // above the first TTD will be delivered to the chain for import. - // - // An alternative would be to check the local chain for exceeding the TTD and - // avoid triggering a sync in that case, but that could also miss sibling or - // other family TTD block being accepted. - if cs.handler.chain.Config().TerminalTotalDifficultyPassed || cs.handler.merger.TDDReached() { - return nil - } - // Ensure we're at minimum peer count. - minPeers := defaultMinSyncPeers - if cs.forced { - minPeers = 1 - } else if minPeers > cs.handler.maxPeers { - minPeers = cs.handler.maxPeers - } - if cs.handler.peers.len() < minPeers { - return nil - } - // We have enough peers, pick the one with the highest TD, but avoid going - // over the terminal total difficulty. Above that we expect the consensus - // clients to direct the chain head to sync to. - peer := cs.handler.peers.peerWithHighestTD() - if peer == nil { - return nil - } - mode, ourTD := cs.modeAndLocalHead() - op := peerToSyncOp(mode, peer) - if op.td.Cmp(ourTD) <= 0 { - // We seem to be in sync according to the legacy rules. In the merge - // world, it can also mean we're stuck on the merge block, waiting for - // a beacon client. In the latter case, notify the user. - if ttd := cs.handler.chain.Config().TerminalTotalDifficulty; ttd != nil && ourTD.Cmp(ttd) >= 0 && time.Since(cs.warned) > 10*time.Second { - log.Warn("Local chain is post-merge, waiting for beacon client sync switch-over...") - cs.warned = time.Now() - } - return nil // We're in sync - } - return op -} - -func peerToSyncOp(mode downloader.SyncMode, p *eth.Peer) *chainSyncOp { - peerHead, peerTD := p.Head() - return &chainSyncOp{mode: mode, peer: p, td: peerTD, head: peerHead} -} - -func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) { - // If we're in snap sync mode, return that directly - if cs.handler.snapSync.Load() { - block := cs.handler.chain.CurrentSnapBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) - return downloader.SnapSync, td - } - // We are probably in full sync, but we might have rewound to before the - // snap sync pivot, check if we should re-enable snap sync. - head := cs.handler.chain.CurrentBlock() - if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil { - if head.Number.Uint64() < *pivot { - block := cs.handler.chain.CurrentSnapBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) - return downloader.SnapSync, td - } - } - // We are in a full sync, but the associated head state is missing. To complete - // the head state, forcefully rerun the snap sync. Note it doesn't mean the - // persistent state is corrupted, just mismatch with the head block. - if !cs.handler.chain.HasState(head.Root) { - block := cs.handler.chain.CurrentSnapBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) - log.Info("Reenabled snap sync as chain is stateless") - return downloader.SnapSync, td - } - // Nope, we're really full syncing - td := cs.handler.chain.GetTd(head.Hash(), head.Number.Uint64()) - return downloader.FullSync, td -} - -// startSync launches doSync in a new goroutine. -func (cs *chainSyncer) startSync(op *chainSyncOp) { - cs.doneCh = make(chan error, 1) - go func() { cs.doneCh <- cs.handler.doSync(op) }() -} - -// doSync synchronizes the local blockchain with a remote peer. -func (h *handler) doSync(op *chainSyncOp) error { - // Run the sync cycle, and disable snap sync if we're past the pivot block - err := h.downloader.LegacySync(op.peer.ID(), op.head, op.td, h.chain.Config().TerminalTotalDifficulty, op.mode) - if err != nil { - return err - } - h.enableSyncedFeatures() - - head := h.chain.CurrentBlock() - if head.Number.Uint64() > 0 { - // We've completed a sync cycle, notify all peers of new state. This path is - // essential in star-topology networks where a gateway node needs to notify - // all its out-of-date peers of the availability of a new block. This failure - // scenario will most often crop up in private and hackathon networks with - // degenerate connectivity, but it should be healthy for the mainnet too to - // more reliably update peers or the local TD state. - if block := h.chain.GetBlock(head.Hash(), head.Number.Uint64()); block != nil { - h.BroadcastBlock(block, false) - } - } - return nil -} diff --git a/eth/sync_test.go b/eth/sync_test.go index a31986730f06..7ede0a82c5d5 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -85,10 +85,11 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) { time.Sleep(250 * time.Millisecond) // Check that snap sync was disabled - op := peerToSyncOp(downloader.SnapSync, empty.handler.peers.peerWithHighestTD()) - if err := empty.handler.doSync(op); err != nil { + if err := empty.handler.downloader.BeaconSync(downloader.SnapSync, full.chain.CurrentBlock(), nil); err != nil { t.Fatal("sync failed:", err) } + empty.handler.enableSyncedFeatures() + if empty.handler.snapSync.Load() { t.Fatalf("snap sync not disabled after successful synchronisation") } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 4d4428f6c63b..0add06c8f69b 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -226,7 +226,7 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf } sub := notifier.CreateSubscription() - resCh := api.traceChain(from, to, config, notifier.Closed()) + resCh := api.traceChain(from, to, config, sub.Err()) go func() { for result := range resCh { notifier.Notify(sub.ID, result) @@ -240,7 +240,7 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf // the end block but excludes the start one. The return value will be one item per // transaction, dependent on the requested tracer. // The tracing procedure should be aborted in case the closed signal is received. -func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed <-chan interface{}) chan *blockTraceResult { +func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed <-chan error) chan *blockTraceResult { reexec := defaultTraceReexec if config != nil && config.Reexec != nil { reexec = *config.Reexec @@ -632,7 +632,6 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat var ( txs = block.Transactions() blockHash = block.Hash() - blockCtx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) signer = types.MakeSigner(api.backend.ChainConfig(), block.Number(), block.Time()) results = make([]*txTraceResult, len(txs)) pend sync.WaitGroup @@ -655,6 +654,11 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat TxIndex: task.index, TxHash: txs[task.index].Hash(), } + // Reconstruct the block context for each transaction + // as the GetHash function of BlockContext is not safe for + // concurrent use. + // See: https://github.com/ethereum/go-ethereum/issues/29114 + blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) res, err := api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config) if err != nil { results[task.index] = &txTraceResult{TxHash: txs[task.index].Hash(), Error: err.Error()} @@ -667,6 +671,7 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat // Feed the transactions into the tracers and return var failed error + blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) txloop: for i, tx := range txs { // Send the trace task over for execution @@ -919,7 +924,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc config.BlockOverrides.Apply(&vmctx) } // Execute the trace - msg, err := args.ToMessage(api.backend.RPCGasCap(), block.BaseFee()) + msg, err := args.ToMessage(api.backend.RPCGasCap(), vmctx.BaseFee) if err != nil { return nil, err } diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index 8a60123dc2c6..38097ff334b2 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" @@ -107,6 +108,11 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { ) defer state.Close() + if test.Genesis.ExcessBlobGas != nil && test.Genesis.BlobGasUsed != nil { + excessBlobGas := eip4844.CalcExcessBlobGas(*test.Genesis.ExcessBlobGas, *test.Genesis.BlobGasUsed) + context.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas) + } + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json new file mode 100644 index 000000000000..315481aff536 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json @@ -0,0 +1,63 @@ +{ + "genesis": { + "baseFeePerGas": "7", + "blobGasUsed": "0", + "difficulty": "0", + "excessBlobGas": "36306944", + "extraData": "0xd983010e00846765746888676f312e32312e308664617277696e", + "gasLimit": "15639172", + "hash": "0xc682259fda061bb9ce8ccb491d5b2d436cb73daf04e1025dd116d045ce4ad28c", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0xae1a5ba939a4c9ac38aabeff361169fb55a6fc2c9511457e0be6eff9514faec0", + "nonce": "0x0000000000000000", + "number": "315", + "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "stateRoot": "0x577f42ab21ccfd946511c57869ace0bdf7c217c36f02b7cd3459df0ed1cffc1a", + "timestamp": "1709626771", + "totalDifficulty": "1", + "withdrawals": [], + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "alloc": { + "0x0000000000000000000000000000000000000000": { + "balance": "0x272e0528" + }, + "0x0c2c51a0990aee1d73c1228de158688341557508": { + "balance": "0xde0b6b3a7640000" + } + }, + "config": { + "chainId": 1337, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "shanghaiTime": 0, + "cancunTime": 0, + "terminalTotalDifficulty": 0, + "terminalTotalDifficultyPassed": true + } + }, + "context": { + "number": "316", + "difficulty": "0", + "timestamp": "1709626785", + "gasLimit": "15654443", + "miner": "0x0000000000000000000000000000000000000000" + }, + "input": "0x03f8b1820539806485174876e800825208940c2c51a0990aee1d73c1228de1586883415575088080c083020000f842a00100c9fbdf97f747e85847b4f3fff408f89c26842f77c882858bf2c89923849aa00138e3896f3c27f2389147507f8bcec52028b0efca6ee842ed83c9158873943880a0dbac3f97a532c9b00e6239b29036245a5bfbb96940b9d848634661abee98b945a03eec8525f261c2e79798f7b45a5d6ccaefa24576d53ba5023e919b86841c0675", + "result": { + "0x0000000000000000000000000000000000000000": { "balance": "0x272e0528" }, + "0x0c2c51a0990aee1d73c1228de158688341557508": { + "balance": "0xde0b6b3a7640000" + } + } +} diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer/create_post_eip158.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer/create_post_eip158.json new file mode 100644 index 000000000000..205b472dabe4 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer/create_post_eip158.json @@ -0,0 +1,64 @@ +{ + "genesis": { + "baseFeePerGas": "7", + "difficulty": "2", + "extraData": "0xd983010d0e846765746888676f312e32312e318664617277696e0000000000001713699f05f79a59abec177c7a87b90ceda79b72ff5edc9197dd7627a447cde45b079bbc3765a236cdf680e2d4d2247135d0e6bb6fd92b50638b92504ddb274f00", + "gasLimit": "30000000", + "hash": "0x6ad5258175c66f4e883d238a92a08428d8ebcbeac631ab7b972634cc05effab3", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "39137", + "stateRoot": "0x715f00df764dbadd4863247a215ac44b5420beafde3ec458b15db7aafa89be0c", + "timestamp": "1709022192", + "totalDifficulty": "78275", + "alloc": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": "2" + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": "64" + } + }, + "config": { + "chainId": 12345, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "clique": { + "period": 5, + "epoch": 30000 + } + } + }, + "context": { + "number": "39138", + "difficulty": "2", + "timestamp": "1709022197", + "gasLimit": "30000000", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0" + }, + "input": "0x02f902af823039408459682f008459682f088302b3538080b90254608060405234801561001057600080fd5b50610234806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806309ce9ccb1461003b5780633fb5c1cb14610059575b600080fd5b610043610075565b60405161005091906100e2565b60405180910390f35b610073600480360381019061006e919061012e565b61007b565b005b60005481565b80600081905550600a8111156100c6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100bd906101de565b60405180910390fd5b50565b6000819050919050565b6100dc816100c9565b82525050565b60006020820190506100f760008301846100d3565b92915050565b600080fd5b61010b816100c9565b811461011657600080fd5b50565b60008135905061012881610102565b92915050565b600060208284031215610144576101436100fd565b5b600061015284828501610119565b91505092915050565b600082825260208201905092915050565b7f4e756d6265722069732067726561746572207468616e2031302c207472616e7360008201527f616374696f6e2072657665727465642e00000000000000000000000000000000602082015250565b60006101c860308361015b565b91506101d38261016c565b604082019050919050565b600060208201905081810360008301526101f7816101bb565b905091905056fea264697066735822122069018995fecf03bda91a88b6eafe41641709dee8b4a706fe301c8a569fe8c1b364736f6c63430008130033c001a0a8cf4729b7e4664687abb3e2559853d7d489eb441519be2a17493061fb4c3a03a04b5a904ba8a6e59c6c40049c4d14a73233aeb8a45b38403199f304630dc0d453", + "result": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": 2 + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": 64 + } + } +} diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_post_eip158.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_post_eip158.json new file mode 100644 index 000000000000..83266f6669cf --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_post_eip158.json @@ -0,0 +1,82 @@ +{ + "genesis": { + "baseFeePerGas": "7", + "difficulty": "2", + "extraData": "0xd983010d0e846765746888676f312e32312e318664617277696e0000000000001713699f05f79a59abec177c7a87b90ceda79b72ff5edc9197dd7627a447cde45b079bbc3765a236cdf680e2d4d2247135d0e6bb6fd92b50638b92504ddb274f00", + "gasLimit": "30000000", + "hash": "0x6ad5258175c66f4e883d238a92a08428d8ebcbeac631ab7b972634cc05effab3", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "39137", + "stateRoot": "0x715f00df764dbadd4863247a215ac44b5420beafde3ec458b15db7aafa89be0c", + "timestamp": "1709022192", + "totalDifficulty": "78275", + "alloc": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": "2" + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": "64" + } + }, + "config": { + "chainId": 12345, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "clique": { + "period": 5, + "epoch": 30000 + } + } + }, + "context": { + "number": "39138", + "difficulty": "2", + "timestamp": "1709022197", + "gasLimit": "30000000", + "miner": "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0" + }, + "input": "0x02f902af823039408459682f008459682f088302b3538080b90254608060405234801561001057600080fd5b50610234806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806309ce9ccb1461003b5780633fb5c1cb14610059575b600080fd5b610043610075565b60405161005091906100e2565b60405180910390f35b610073600480360381019061006e919061012e565b61007b565b005b60005481565b80600081905550600a8111156100c6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100bd906101de565b60405180910390fd5b50565b6000819050919050565b6100dc816100c9565b82525050565b60006020820190506100f760008301846100d3565b92915050565b600080fd5b61010b816100c9565b811461011657600080fd5b50565b60008135905061012881610102565b92915050565b600060208284031215610144576101436100fd565b5b600061015284828501610119565b91505092915050565b600082825260208201905092915050565b7f4e756d6265722069732067726561746572207468616e2031302c207472616e7360008201527f616374696f6e2072657665727465642e00000000000000000000000000000000602082015250565b60006101c860308361015b565b91506101d38261016c565b604082019050919050565b600060208201905081810360008301526101f7816101bb565b905091905056fea264697066735822122069018995fecf03bda91a88b6eafe41641709dee8b4a706fe301c8a569fe8c1b364736f6c63430008130033c001a0a8cf4729b7e4664687abb3e2559853d7d489eb441519be2a17493061fb4c3a03a04b5a904ba8a6e59c6c40049c4d14a73233aeb8a45b38403199f304630dc0d453", + "tracerConfig": { + "diffMode": true + }, + "result": { + "post": { + "0x1bda2f8e4735507930bd6cfe873bf0bf0f4ab1de": { + "code": "0x608060405234801561001057600080fd5b50600436106100365760003560e01c806309ce9ccb1461003b5780633fb5c1cb14610059575b600080fd5b610043610075565b60405161005091906100e2565b60405180910390f35b610073600480360381019061006e919061012e565b61007b565b005b60005481565b80600081905550600a8111156100c6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100bd906101de565b60405180910390fd5b50565b6000819050919050565b6100dc816100c9565b82525050565b60006020820190506100f760008301846100d3565b92915050565b600080fd5b61010b816100c9565b811461011657600080fd5b50565b60008135905061012881610102565b92915050565b600060208284031215610144576101436100fd565b5b600061015284828501610119565b91505092915050565b600082825260208201905092915050565b7f4e756d6265722069732067726561746572207468616e2031302c207472616e7360008201527f616374696f6e2072657665727465642e00000000000000000000000000000000602082015250565b60006101c860308361015b565b91506101d38261016c565b604082019050919050565b600060208201905081810360008301526101f7816101bb565b905091905056fea264697066735822122069018995fecf03bda91a88b6eafe41641709dee8b4a706fe301c8a569fe8c1b364736f6c63430008130033", + "nonce": 1 + }, + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f0645688331eb5690" + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6aae9b21b6ee855", + "nonce": 65 + } + }, + "pre": { + "0x2445e8c26a2bf3d1e59f1bb9b1d442caf90768e0": { + "balance": "0x10f06447a8d44dba190", + "nonce": 2 + }, + "0x82211934c340b29561381392348d48413e15adc8": { + "balance": "0x6abd7a808913ed2", + "nonce": 64 + } + } + } +} diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index f85cf6206a72..be9b58a4cd3c 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -161,7 +161,7 @@ func (t *callTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, sco return } // Avoid processing nested calls when only caring about top call - if t.config.OnlyTopCall && depth > 0 { + if t.config.OnlyTopCall && depth > 1 { return } // Skip if tracing was interrupted diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index d7e10173cf27..b86c5c461c7d 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" ) //go:generate go run github.com/fjl/gencodec -type account -field-override accountMarshaling -out gen_account_json.go @@ -102,6 +103,9 @@ func (t *prestateTracer) CaptureStart(env *vm.EVM, from common.Address, to commo // The recipient balance includes the value transferred. toBal := new(big.Int).Sub(t.pre[to].Balance, value) t.pre[to].Balance = toBal + if env.ChainConfig().Rules(env.Context.BlockNumber, env.Context.Random != nil, env.Context.Time).IsEIP158 && create { + t.pre[to].Nonce-- + } // The sender balance is after reducing: value and gasLimit. // We need to re-add them to get the pre-tx balance. @@ -109,6 +113,12 @@ func (t *prestateTracer) CaptureStart(env *vm.EVM, from common.Address, to commo gasPrice := env.TxContext.GasPrice consumedGas := new(big.Int).Mul(gasPrice, new(big.Int).SetUint64(t.gasLimit)) fromBal.Add(fromBal, new(big.Int).Add(value, consumedGas)) + + // Add blob fee to the sender's balance. + if env.Context.BlobBaseFee != nil && len(env.TxContext.BlobHashes) > 0 { + blobGas := uint64(params.BlobTxBlobGasPerBlob * len(env.TxContext.BlobHashes)) + fromBal.Add(fromBal, new(big.Int).Mul(env.Context.BlobBaseFee, new(big.Int).SetUint64(blobGas))) + } t.pre[from].Balance = fromBal t.pre[from].Nonce-- diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 0d2675f8d10d..2f3229cedcb5 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -602,17 +602,22 @@ func testAtFunctions(t *testing.T, client *rpc.Client) { } // send a transaction for some interesting pending status + // and wait for the transaction to be included in the pending block sendTransaction(ec) - time.Sleep(100 * time.Millisecond) - // Check pending transaction count - pending, err := ec.PendingTransactionCount(context.Background()) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if pending != 1 { - t.Fatalf("unexpected pending, wanted 1 got: %v", pending) + // wait for the transaction to be included in the pending block + for { + // Check pending transaction count + pending, err := ec.PendingTransactionCount(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if pending == 1 { + break + } + time.Sleep(100 * time.Millisecond) } + // Query balance balance, err := ec.BalanceAt(context.Background(), testAddr, nil) if err != nil { @@ -737,7 +742,7 @@ func sendTransaction(ec *Client) error { if err != nil { return err } - nonce, err := ec.PendingNonceAt(context.Background(), testAddr) + nonce, err := ec.NonceAt(context.Background(), testAddr, nil) if err != nil { return err } diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 73d05d499efe..b1678b67664e 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -245,6 +245,12 @@ func toCallArg(msg ethereum.CallMsg) interface{} { if msg.AccessList != nil { arg["accessList"] = msg.AccessList } + if msg.BlobGasFeeCap != nil { + arg["maxFeePerBlobGas"] = (*hexutil.Big)(msg.BlobGasFeeCap) + } + if msg.BlobHashes != nil { + arg["blobVersionedHashes"] = msg.BlobHashes + } return arg } diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 158886475eda..d562bcda1f01 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -146,7 +146,7 @@ func TestGethClient(t *testing.T) { func(t *testing.T) { testCallContractWithBlockOverrides(t, client) }, }, // The testaccesslist is a bit time-sensitive: the newTestBackend imports - // one block. The `testAcessList` fails if the miner has not yet created a + // one block. The `testAccessList` fails if the miner has not yet created a // new pending-block after the import event. // Hence: this test should be last, execute the tests serially. { diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index af4686cf5b72..57689ab04b9b 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -589,8 +589,8 @@ func (b *batch) Reset() { func (b *batch) Replay(w ethdb.KeyValueWriter) error { reader := b.b.Reader() for { - kind, k, v, ok := reader.Next() - if !ok { + kind, k, v, ok, err := reader.Next() + if !ok || err != nil { break } // The (k,v) slices might be overwritten if the batch is reset/reused, diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 61ceec443ebc..6e71666ec121 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -39,7 +39,6 @@ import ( ethproto "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" @@ -80,13 +79,6 @@ type fullNodeBackend interface { SuggestGasTipCap(ctx context.Context) (*big.Int, error) } -// miningNodeBackend encompasses the functionality necessary for a mining node -// reporting to ethstats -type miningNodeBackend interface { - fullNodeBackend - Miner() *miner.Miner -} - // Service implements an Ethereum netstats reporting daemon that pushes local // chain statistics up to a monitoring server. type Service struct { @@ -777,30 +769,21 @@ func (s *Service) reportPending(conn *connWrapper) error { type nodeStats struct { Active bool `json:"active"` Syncing bool `json:"syncing"` - Mining bool `json:"mining"` - Hashrate int `json:"hashrate"` Peers int `json:"peers"` GasPrice int `json:"gasPrice"` Uptime int `json:"uptime"` } -// reportStats retrieves various stats about the node at the networking and -// mining layer and reports it to the stats server. +// reportStats retrieves various stats about the node at the networking layer +// and reports it to the stats server. func (s *Service) reportStats(conn *connWrapper) error { - // Gather the syncing and mining infos from the local miner instance + // Gather the syncing infos from the local miner instance var ( - mining bool - hashrate int syncing bool gasprice int ) // check if backend is a full node if fullBackend, ok := s.backend.(fullNodeBackend); ok { - if miningBackend, ok := s.backend.(miningNodeBackend); ok { - mining = miningBackend.Miner().Mining() - hashrate = int(miningBackend.Miner().Hashrate()) - } - sync := fullBackend.SyncProgress() syncing = !sync.Done() @@ -820,8 +803,6 @@ func (s *Service) reportStats(conn *connWrapper) error { "id": s.node, "stats": &nodeStats{ Active: true, - Mining: mining, - Hashrate: hashrate, Peers: s.server.PeerCount(), GasPrice: gasprice, Syncing: syncing, diff --git a/go.mod b/go.mod index 7a54b1ff7ca9..4e481065469f 100644 --- a/go.mod +++ b/go.mod @@ -13,14 +13,15 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.2.0 github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.79.0 - github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 + github.com/cockroachdb/pebble v1.1.0 github.com/consensys/gnark-crypto v0.12.1 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 - github.com/crate-crypto/go-kzg-4844 v0.7.0 + github.com/crate-crypto/go-kzg-4844 v1.0.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 - github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 - github.com/ethereum/c-kzg-4844 v0.4.0 + github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 + github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 + github.com/ethereum/c-kzg-4844 v1.0.0 github.com/fatih/color v1.13.0 github.com/ferranbt/fastssz v0.1.2 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e @@ -46,7 +47,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 github.com/julienschmidt/httprouter v1.3.0 - github.com/karalabe/usb v0.0.2 + github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.17 @@ -54,6 +55,8 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 + github.com/protolambda/zrnt v0.30.0 + github.com/protolambda/ztyp v0.2.2 github.com/rs/cors v1.7.0 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.2.0 @@ -63,10 +66,10 @@ require ( github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.25.7 go.uber.org/automaxprocs v1.5.2 - golang.org/x/crypto v0.17.0 + golang.org/x/crypto v0.21.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/sync v0.5.0 - golang.org/x/sys v0.16.0 + golang.org/x/sys v0.18.0 golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.15.0 @@ -91,10 +94,9 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cockroachdb/errors v1.8.1 // indirect - github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect - github.com/cockroachdb/redact v1.0.8 // indirect - github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect + github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -102,11 +104,11 @@ require ( github.com/deepmap/oapi-codegen v1.6.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -139,8 +141,8 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.18.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + golang.org/x/net v0.21.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index bb4ded5c2ff7..b5cb268a0bda 100644 --- a/go.sum +++ b/go.sum @@ -31,7 +31,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= @@ -44,20 +43,14 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkM github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= -github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -65,7 +58,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= @@ -92,7 +84,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwF github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -119,36 +110,27 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.79.0 h1:ErwCYDjFCYppDJlDJ/5WhsSmzegAUe2+K9qgFyQDg3M= github.com/cloudflare/cloudflare-go v0.79.0/go.mod h1:gkHQf9xEubaQPEuerBuoinR9P8bf8a05Lq0X6WKy1Oc= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= -github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= -github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= -github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= -github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= -github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= -github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= +github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= -github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= -github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -162,57 +144,48 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/deepmap/oapi-codegen v1.6.0 h1:w/d1ntwh91XI0b/8ja7+u5SvA4IFfM0UNNLmiDR1gg0= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao= +github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 h1:qwcF+vdFrvPSEUDSX5RVoRccG8a5DhOdWdQ4zN62zzo= -github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 h1:+3HCtB74++ClLy8GgjUQYeC8R4ILzVcIe8+5edAJJnE= +github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= -github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk= github.com/ferranbt/fastssz v0.1.2/go.mod h1:X5UPrE2u1UJjxHA8X54u04SBwdAQjG2sFtWs39YxyWs= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 h1:IZqZOB2fydHte3kUgxrzK5E1fW7RQGeDwE8F/ZZnUYc= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= -github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2GihuqhwdILrV+7GJel5lyPV3u1+PgzrWLc0TkE= github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -222,7 +195,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -231,20 +203,13 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -276,11 +241,11 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= -github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -295,8 +260,6 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -319,9 +282,7 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= @@ -334,34 +295,26 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6wn4Ej8vjuVGxeHdan+bRb2ebyv4= github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb-client-go/v2 v2.4.0 h1:HGBfZYStlx3Kqvsv1h2pJixbCl/jhnFtxpKFAv9Tu5k= github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c h1:qSHzRbhzK8RdXOsAdfDgO49TtqC1oZ+acxPrkfTxcCs= github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= -github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= -github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= -github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY= @@ -377,30 +330,17 @@ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= -github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= -github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= -github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= -github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 h1:msKODTL1m0wigztaqILOtla9HeW1ciscYG4xjLtvk5I= +github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -418,11 +358,9 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= @@ -432,7 +370,6 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -444,17 +381,12 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= -github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= -github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= @@ -467,23 +399,18 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= -github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= -github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -491,11 +418,9 @@ github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -528,8 +453,14 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e/go.mod h1:MPZvj2Pr0N8/dXyTPS5REeg2sdLG7t8DRzC1rLv925w= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 h1:cZC+usqsYgHtlBaGulVnZ1hfKAi8iWtujBnRLQE698c= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7/go.mod h1:IToEjHuttnUzwZI5KBSM/LOOW3qLbbrHOEfp3SbECGY= +github.com/protolambda/messagediff v1.4.0/go.mod h1:LboJp0EwIbJsePYpzh5Op/9G1/4mIztMRYzzwR0dR2M= +github.com/protolambda/zrnt v0.30.0 h1:pHEn69ZgaDFGpLGGYG1oD7DvYI7RDirbMBPfbC+8p4g= +github.com/protolambda/zrnt v0.30.0/go.mod h1:qcdX9CXFeVNCQK/q0nswpzhd+31RHMk2Ax/2lMsJ4Jw= +github.com/protolambda/ztyp v0.2.2 h1:rVcL3vBu9W/aV646zF6caLS/dyn9BN8NYiuJzicLNyY= +github.com/protolambda/ztyp v0.2.2/go.mod h1:9bYgKGqg3wJqT9ac1gI2hnVb0STQq7p/1lapqrqY1dU= github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220714111606-acbb2962fb48 h1:cSo6/vk8YpvkLbk9v3FO97cakNmUoxwi2KMP8hd5WIw= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -539,26 +470,13 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -580,26 +498,13 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -613,18 +518,16 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -664,11 +567,9 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -677,7 +578,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -699,8 +599,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -725,7 +625,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -735,7 +634,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -784,8 +682,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -808,15 +706,11 @@ golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -883,7 +777,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -913,7 +806,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -938,8 +830,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -948,9 +840,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -964,6 +853,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 1dda10205822..f3f9d1778ab0 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -452,6 +453,7 @@ func newGQLService(t *testing.T, stack *node.Node, shanghai bool, gspec *core.Ge TrieDirtyCache: 5, TrieTimeout: 60 * time.Minute, SnapshotCache: 5, + StateScheme: rawdb.HashScheme, } var engine consensus.Engine = ethash.NewFaker() if shanghai { diff --git a/internal/build/download.go b/internal/build/download.go index fda573df8331..206c51dce1e3 100644 --- a/internal/build/download.go +++ b/internal/build/download.go @@ -84,10 +84,12 @@ func (db *ChecksumDB) DownloadFile(url, dstPath string) error { resp, err := http.Get(url) if err != nil { return fmt.Errorf("download error: %v", err) - } else if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download error: status %d", resp.StatusCode) } defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download error: status %d", resp.StatusCode) + } if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { return err } diff --git a/internal/era/accumulator.go b/internal/era/accumulator.go index 19e03973f1f5..2ece2755e12d 100644 --- a/internal/era/accumulator.go +++ b/internal/era/accumulator.go @@ -17,6 +17,7 @@ package era import ( + "errors" "fmt" "math/big" @@ -28,7 +29,7 @@ import ( // accumulator of header records. func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, error) { if len(hashes) != len(tds) { - return common.Hash{}, fmt.Errorf("must have equal number hashes as td values") + return common.Hash{}, errors.New("must have equal number hashes as td values") } if len(hashes) > MaxEra1Size { return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEra1Size) diff --git a/internal/era/builder.go b/internal/era/builder.go index 9217c049f33b..75782a08c251 100644 --- a/internal/era/builder.go +++ b/internal/era/builder.go @@ -18,6 +18,7 @@ package era import ( "bytes" "encoding/binary" + "errors" "fmt" "io" "math/big" @@ -158,7 +159,7 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm // corresponding e2store entries. func (b *Builder) Finalize() (common.Hash, error) { if b.startNum == nil { - return common.Hash{}, fmt.Errorf("finalize called on empty builder") + return common.Hash{}, errors.New("finalize called on empty builder") } // Compute accumulator root and write entry. root, err := ComputeAccumulator(b.hashes, b.tds) diff --git a/internal/era/e2store/e2store.go b/internal/era/e2store/e2store.go index d85b3e44e97d..8e4d5dd24a55 100644 --- a/internal/era/e2store/e2store.go +++ b/internal/era/e2store/e2store.go @@ -18,6 +18,7 @@ package e2store import ( "encoding/binary" + "errors" "fmt" "io" ) @@ -160,7 +161,7 @@ func (r *Reader) ReadMetadataAt(off int64) (typ uint16, length uint32, err error // Check reserved bytes of header. if b[6] != 0 || b[7] != 0 { - return 0, 0, fmt.Errorf("reserved bytes are non-zero") + return 0, 0, errors.New("reserved bytes are non-zero") } return typ, length, nil diff --git a/internal/era/e2store/e2store_test.go b/internal/era/e2store/e2store_test.go index febcffe4cf2c..b0803493c7cd 100644 --- a/internal/era/e2store/e2store_test.go +++ b/internal/era/e2store/e2store_test.go @@ -18,7 +18,7 @@ package e2store import ( "bytes" - "fmt" + "errors" "io" "testing" @@ -92,7 +92,7 @@ func TestDecode(t *testing.T) { }, { // basic invalid decoding have: "ffff000000000001", - err: fmt.Errorf("reserved bytes are non-zero"), + err: errors.New("reserved bytes are non-zero"), }, { // no more entries to read, returns EOF have: "", diff --git a/internal/era/era.go b/internal/era/era.go index a0e701b7e0f9..2099c2d575c7 100644 --- a/internal/era/era.go +++ b/internal/era/era.go @@ -18,6 +18,7 @@ package era import ( "encoding/binary" + "errors" "fmt" "io" "math/big" @@ -127,7 +128,7 @@ func (e *Era) Close() error { func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds") + return nil, errors.New("out-of-bounds") } off, err := e.readOffset(num) if err != nil { diff --git a/internal/era/iterator.go b/internal/era/iterator.go index e74a8154b1a6..d90e9586a4e6 100644 --- a/internal/era/iterator.go +++ b/internal/era/iterator.go @@ -17,6 +17,7 @@ package era import ( + "errors" "fmt" "io" "math/big" @@ -30,7 +31,7 @@ type Iterator struct { inner *RawIterator } -// NewRawIterator returns a new Iterator instance. Next must be immediately +// NewIterator returns a new Iterator instance. Next must be immediately // called on new iterators to load the first item. func NewIterator(e *Era) (*Iterator, error) { inner, err := NewRawIterator(e) @@ -61,7 +62,7 @@ func (it *Iterator) Error() error { // Block returns the block for the iterator's current position. func (it *Iterator) Block() (*types.Block, error) { if it.inner.Header == nil || it.inner.Body == nil { - return nil, fmt.Errorf("header and body must be non-nil") + return nil, errors.New("header and body must be non-nil") } var ( header types.Header diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index df25dfbd37a6..863849f4da6a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -288,7 +288,7 @@ type PersonalAccountAPI struct { b Backend } -// NewPersonalAccountAPI create a new PersonalAccountAPI. +// NewPersonalAccountAPI creates a new PersonalAccountAPI. func NewPersonalAccountAPI(b Backend, nonceLock *AddrLocker) *PersonalAccountAPI { return &PersonalAccountAPI{ am: b.AccountManager(), @@ -453,7 +453,7 @@ func (s *PersonalAccountAPI) signTransaction(ctx context.Context, args *Transact return nil, err } // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Assemble the transaction and sign with the wallet @@ -1093,14 +1093,14 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S defer cancel() // Get a new instance of the EVM. - msg, err := args.ToMessage(globalGasCap, header.BaseFee) - if err != nil { - return nil, err - } blockCtx := core.NewEVMBlockContext(header, NewChainContext(ctx, b), nil) if blockOverrides != nil { blockOverrides.Apply(&blockCtx) } + msg, err := args.ToMessage(globalGasCap, blockCtx.BaseFee) + if err != nil { + return nil, err + } evm := b.GetEVM(ctx, msg, state, header, &vm.Config{NoBaseFee: true}, &blockCtx) // Wait for the context to be done and cancel the evm. Even if the @@ -1485,14 +1485,9 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if db == nil || err != nil { return nil, 0, nil, err } - // If the gas amount is not set, default to RPC gas cap. - if args.Gas == nil { - tmp := hexutil.Uint64(b.RPCGasCap()) - args.Gas = &tmp - } // Ensure any missing fields are filled, extract the recipient and input data - if err := args.setDefaults(ctx, b); err != nil { + if err := args.setDefaults(ctx, b, true); err != nil { return nil, 0, nil, err } var to common.Address @@ -1795,7 +1790,7 @@ func (s *TransactionAPI) SendTransaction(ctx context.Context, args TransactionAr } // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return common.Hash{}, err } // Assemble the transaction and sign with the wallet @@ -1815,7 +1810,7 @@ func (s *TransactionAPI) FillTransaction(ctx context.Context, args TransactionAr args.blobSidecarAllowed = true // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Assemble the transaction and obtain rlp @@ -1884,7 +1879,7 @@ func (s *TransactionAPI) SignTransaction(ctx context.Context, args TransactionAr if args.Nonce == nil { return nil, errors.New("nonce not specified") } - if err := args.setDefaults(ctx, s.b); err != nil { + if err := args.setDefaults(ctx, s.b, false); err != nil { return nil, err } // Before actually sign the transaction, ensure the transaction fee is reasonable. @@ -1933,7 +1928,7 @@ func (s *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, g if sendArgs.Nonce == nil { return common.Hash{}, errors.New("missing transaction nonce in transaction spec") } - if err := sendArgs.setDefaults(ctx, s.b); err != nil { + if err := sendArgs.setDefaults(ctx, s.b, false); err != nil { return common.Hash{}, err } matchTx := sendArgs.toTransaction() diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 8a2e367f4a83..5636309589df 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -547,7 +547,7 @@ func (b testBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOr } panic("only implemented for number") } -func (b testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { panic("implement me") } +func (b testBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { panic("implement me") } func (b testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { header, err := b.HeaderByHash(ctx, hash) if header == nil || err != nil { @@ -615,9 +615,6 @@ func (b testBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { panic("implement me") } -func (b testBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - panic("implement me") -} func (b testBackend) BloomStatus() (uint64, uint64) { panic("implement me") } func (b testBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { panic("implement me") @@ -1091,7 +1088,8 @@ func TestFillBlobTransaction(t *testing.T) { Config: params.MergedTestChainConfig, Alloc: types.GenesisAlloc{}, } - emptyBlob = kzg4844.Blob{} + emptyBlob = new(kzg4844.Blob) + emptyBlobs = []kzg4844.Blob{*emptyBlob} emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) emptyBlobHash common.Hash = kzg4844.CalcBlobHashV1(sha256.New(), &emptyBlobCommit) @@ -1174,14 +1172,14 @@ func TestFillBlobTransaction(t *testing.T) { From: &b.acc.Address, To: &to, Value: (*hexutil.Big)(big.NewInt(1)), - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, want: &result{ Hashes: []common.Hash{emptyBlobHash}, Sidecar: &types.BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, @@ -1194,14 +1192,14 @@ func TestFillBlobTransaction(t *testing.T) { To: &to, Value: (*hexutil.Big)(big.NewInt(1)), BlobHashes: []common.Hash{emptyBlobHash}, - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, want: &result{ Hashes: []common.Hash{emptyBlobHash}, Sidecar: &types.BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, @@ -1214,7 +1212,7 @@ func TestFillBlobTransaction(t *testing.T) { To: &to, Value: (*hexutil.Big)(big.NewInt(1)), BlobHashes: []common.Hash{{0x01, 0x22}}, - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, @@ -1226,12 +1224,12 @@ func TestFillBlobTransaction(t *testing.T) { From: &b.acc.Address, To: &to, Value: (*hexutil.Big)(big.NewInt(1)), - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, }, want: &result{ Hashes: []common.Hash{emptyBlobHash}, Sidecar: &types.BlobTxSidecar{ - Blobs: []kzg4844.Blob{emptyBlob}, + Blobs: emptyBlobs, Commitments: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}, }, @@ -1244,7 +1242,7 @@ func TestFillBlobTransaction(t *testing.T) { if len(tc.err) > 0 { if err == nil { t.Fatalf("missing error. want: %s", tc.err) - } else if err != nil && err.Error() != tc.err { + } else if err.Error() != tc.err { t.Fatalf("error mismatch. want: %s, have: %s", tc.err, err.Error()) } return @@ -1272,10 +1270,14 @@ func TestFillBlobTransaction(t *testing.T) { func argsFromTransaction(tx *types.Transaction, from common.Address) TransactionArgs { var ( - gas = tx.Gas() - nonce = tx.Nonce() - input = tx.Data() + gas = tx.Gas() + nonce = tx.Nonce() + input = tx.Data() + accessList *types.AccessList ) + if acl := tx.AccessList(); acl != nil { + accessList = &acl + } return TransactionArgs{ From: &from, To: tx.To(), @@ -1286,10 +1288,9 @@ func argsFromTransaction(tx *types.Transaction, from common.Address) Transaction Nonce: (*hexutil.Uint64)(&nonce), Input: (*hexutil.Bytes)(&input), ChainID: (*hexutil.Big)(tx.ChainId()), - // TODO: impl accessList conversion - //AccessList: tx.AccessList(), - BlobFeeCap: (*hexutil.Big)(tx.BlobGasFeeCap()), - BlobHashes: tx.BlobHashes(), + AccessList: accessList, + BlobFeeCap: (*hexutil.Big)(tx.BlobGasFeeCap()), + BlobHashes: tx.BlobHashes(), } } @@ -1818,6 +1819,7 @@ func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Ha tx *types.Transaction err error ) + b.SetPoS() switch i { case 0: // transfer 1000wei @@ -1866,7 +1868,6 @@ func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Ha b.AddTx(tx) txHashes[i] = tx.Hash() } - b.SetPoS() }) return backend, txHashes } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 5f408ba20ba5..fd2f5699eabf 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -65,7 +65,7 @@ type Backend interface { BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) + Pending() (*types.Block, types.Receipts, *state.StateDB) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) GetTd(ctx context.Context, hash common.Hash) *big.Int GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM @@ -94,7 +94,6 @@ type Backend interface { GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription BloomStatus() (uint64, uint64) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) } diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 03ffb7524f59..2751d5b5aae6 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -96,7 +96,7 @@ func (args *TransactionArgs) data() []byte { } // setDefaults fills in default values for unspecified tx fields. -func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { +func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, skipGasEstimation bool) error { if err := args.setBlobTxSidecar(ctx, b); err != nil { return err } @@ -136,30 +136,37 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { } } - // Estimate the gas usage if necessary. if args.Gas == nil { - // These fields are immutable during the estimation, safe to - // pass the pointer directly. - data := args.data() - callArgs := TransactionArgs{ - From: args.From, - To: args.To, - GasPrice: args.GasPrice, - MaxFeePerGas: args.MaxFeePerGas, - MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, - Value: args.Value, - Data: (*hexutil.Bytes)(&data), - AccessList: args.AccessList, - BlobFeeCap: args.BlobFeeCap, - BlobHashes: args.BlobHashes, - } - latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) - estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, b.RPCGasCap()) - if err != nil { - return err + if skipGasEstimation { // Skip gas usage estimation if a precise gas limit is not critical, e.g., in non-transaction calls. + gas := hexutil.Uint64(b.RPCGasCap()) + if gas == 0 { + gas = hexutil.Uint64(math.MaxUint64 / 2) + } + args.Gas = &gas + } else { // Estimate the gas usage otherwise. + // These fields are immutable during the estimation, safe to + // pass the pointer directly. + data := args.data() + callArgs := TransactionArgs{ + From: args.From, + To: args.To, + GasPrice: args.GasPrice, + MaxFeePerGas: args.MaxFeePerGas, + MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, + Value: args.Value, + Data: (*hexutil.Bytes)(&data), + AccessList: args.AccessList, + BlobFeeCap: args.BlobFeeCap, + BlobHashes: args.BlobHashes, + } + latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, b.RPCGasCap()) + if err != nil { + return err + } + args.Gas = &estimated + log.Trace("Estimate gas usage automatically", "gas", args.Gas) } - args.Gas = &estimated - log.Trace("Estimate gas usage automatically", "gas", args.Gas) } // If chain id is provided, ensure it matches the local chain id. Otherwise, set the local @@ -177,6 +184,14 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { // setFeeDefaults fills in default fee values for unspecified tx fields. func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) error { + head := b.CurrentHeader() + // Sanity check the EIP-4844 fee parameters. + if args.BlobFeeCap != nil && args.BlobFeeCap.ToInt().Sign() == 0 { + return errors.New("maxFeePerBlobGas, if specified, must be non-zero") + } + if err := args.setCancunFeeDefaults(ctx, head, b); err != nil { + return err + } // If both gasPrice and at least one of the EIP-1559 fee parameters are specified, error. if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") @@ -186,7 +201,6 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro // other tx values. See https://github.com/ethereum/go-ethereum/pull/23274 // for more information. eip1559ParamsSet := args.MaxFeePerGas != nil && args.MaxPriorityFeePerGas != nil - // Sanity check the EIP-1559 fee parameters if present. if args.GasPrice == nil && eip1559ParamsSet { if args.MaxFeePerGas.ToInt().Sign() == 0 { @@ -198,13 +212,7 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro return nil // No need to set anything, user already set MaxFeePerGas and MaxPriorityFeePerGas } - // Sanity check the EIP-4844 fee parameters. - if args.BlobFeeCap != nil && args.BlobFeeCap.ToInt().Sign() == 0 { - return errors.New("maxFeePerBlobGas must be non-zero") - } - // Sanity check the non-EIP-1559 fee parameters. - head := b.CurrentHeader() isLondon := b.ChainConfig().IsLondon(head.Number) if args.GasPrice != nil && !eip1559ParamsSet { // Zero gas-price is not allowed after London fork @@ -215,21 +223,14 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro } // Now attempt to fill in default value depending on whether London is active or not. - if b.ChainConfig().IsCancun(head.Number, head.Time) { - if err := args.setCancunFeeDefaults(ctx, head, b); err != nil { - return err - } - } else if isLondon { - if args.BlobFeeCap != nil { - return errors.New("maxFeePerBlobGas is not valid before Cancun is active") - } + if isLondon { // London is active, set maxPriorityFeePerGas and maxFeePerGas. if err := args.setLondonFeeDefaults(ctx, head, b); err != nil { return err } } else { - if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil || args.BlobFeeCap != nil { - return errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active") + if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { + return errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active") } // London not active, set gas price. price, err := b.SuggestGasTipCap(ctx) @@ -245,15 +246,19 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro func (args *TransactionArgs) setCancunFeeDefaults(ctx context.Context, head *types.Header, b Backend) error { // Set maxFeePerBlobGas if it is missing. if args.BlobHashes != nil && args.BlobFeeCap == nil { + var excessBlobGas uint64 + if head.ExcessBlobGas != nil { + excessBlobGas = *head.ExcessBlobGas + } // ExcessBlobGas must be set for a Cancun block. - blobBaseFee := eip4844.CalcBlobFee(*head.ExcessBlobGas) + blobBaseFee := eip4844.CalcBlobFee(excessBlobGas) // Set the max fee to be 2 times larger than the previous block's blob base fee. // The additional slack allows the tx to not become invalidated if the base // fee is rising. val := new(big.Int).Mul(blobBaseFee, big.NewInt(2)) args.BlobFeeCap = (*hexutil.Big)(val) } - return args.setLondonFeeDefaults(ctx, head, b) + return nil } // setLondonFeeDefaults fills in reasonable default fee values for unspecified fields. @@ -321,12 +326,12 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, b Backend) er commitments := make([]kzg4844.Commitment, n) proofs := make([]kzg4844.Proof, n) for i, b := range args.Blobs { - c, err := kzg4844.BlobToCommitment(b) + c, err := kzg4844.BlobToCommitment(&b) if err != nil { return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) } commitments[i] = c - p, err := kzg4844.ComputeBlobProof(b, c) + p, err := kzg4844.ComputeBlobProof(&b, c) if err != nil { return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) } @@ -336,7 +341,7 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, b Backend) er args.Proofs = proofs } else { for i, b := range args.Blobs { - if err := kzg4844.VerifyBlobProof(b, args.Commitments[i], args.Proofs[i]); err != nil { + if err := kzg4844.VerifyBlobProof(&b, args.Commitments[i], args.Proofs[i]); err != nil { return fmt.Errorf("failed to verify blob proof: %v", err) } } diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index f0fdb6d8ee2d..24ecb1dee4e7 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -153,14 +153,14 @@ func TestSetFeeDefaults(t *testing.T) { "legacy", &TransactionArgs{MaxFeePerGas: maxFee}, nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), + errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), }, { "dynamic fee tx pre-London, priorityFee set", "legacy", &TransactionArgs{MaxPriorityFeePerGas: fortytwo}, nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), + errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), }, { "dynamic fee tx, maxFee < priorityFee", @@ -207,20 +207,6 @@ func TestSetFeeDefaults(t *testing.T) { errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), }, // EIP-4844 - { - "set maxFeePerBlobGas pre cancun", - "london", - &TransactionArgs{BlobFeeCap: fortytwo}, - nil, - errors.New("maxFeePerBlobGas is not valid before Cancun is active"), - }, - { - "set maxFeePerBlobGas pre london", - "legacy", - &TransactionArgs{BlobFeeCap: fortytwo}, - nil, - errors.New("maxFeePerGas and maxPriorityFeePerGas and maxFeePerBlobGas are not valid before London is active"), - }, { "set gas price and maxFee for blob transaction", "cancun", @@ -235,6 +221,13 @@ func TestSetFeeDefaults(t *testing.T) { &TransactionArgs{BlobHashes: []common.Hash{}, BlobFeeCap: (*hexutil.Big)(big.NewInt(4)), MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, nil, }, + { + "fill maxFeePerBlobGas when dynamic fees are set", + "cancun", + &TransactionArgs{BlobHashes: []common.Hash{}, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + &TransactionArgs{BlobHashes: []common.Hash{}, BlobFeeCap: (*hexutil.Big)(big.NewInt(4)), MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, } ctx := context.Background() @@ -244,11 +237,16 @@ func TestSetFeeDefaults(t *testing.T) { } got := test.in err := got.setFeeDefaults(ctx, b) - if err != nil && err.Error() == test.err.Error() { - // Test threw expected error. + if err != nil { + if test.err == nil { + t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } else if err.Error() != test.err.Error() { + t.Fatalf("test %d (%s): unexpected error: (got: %s, want: %s)", i, test.name, err, test.err) + } + // Matching error. continue - } else if err != nil { - t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } else if test.err != nil { + t.Fatalf("test %d (%s): expected error: %s", i, test.name, test.err) } if !reflect.DeepEqual(got, test.want) { t.Fatalf("test %d (%s): did not fill defaults as expected: (got: %v, want: %v)", i, test.name, got, test.want) @@ -360,7 +358,7 @@ func (b *backendMock) StateAndHeaderByNumber(ctx context.Context, number rpc.Blo func (b *backendMock) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { return nil, nil, nil } -func (b *backendMock) PendingBlockAndReceipts() (*types.Block, types.Receipts) { return nil, nil } +func (b *backendMock) Pending() (*types.Block, types.Receipts, *state.StateDB) { return nil, nil, nil } func (b *backendMock) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { return nil, nil } @@ -398,9 +396,6 @@ func (b *backendMock) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscr func (b *backendMock) BloomStatus() (uint64, uint64) { return 0, 0 } func (b *backendMock) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {} func (b *backendMock) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return nil } -func (b *backendMock) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { - return nil -} func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return nil } diff --git a/internal/flags/categories.go b/internal/flags/categories.go index 3ff0767921b9..d426add55b10 100644 --- a/internal/flags/categories.go +++ b/internal/flags/categories.go @@ -20,7 +20,7 @@ import "github.com/urfave/cli/v2" const ( EthCategory = "ETHEREUM" - LightCategory = "LIGHT CLIENT" + BeaconCategory = "BEACON CHAIN" DevCategory = "DEVELOPER CHAIN" StateCategory = "STATE HISTORY MANAGEMENT" TxPoolCategory = "TRANSACTION POOL (EVM)" diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index 0b360e74150a..4196cb8db0ee 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -3734,7 +3734,7 @@ var inputCallFormatter = function (options){ options.to = inputAddressFormatter(options.to); } - ['maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { + ['maxFeePerBlobGas', 'maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { return options[key] !== undefined; }).forEach(function(key){ options[key] = utils.fromDecimal(options[key]); @@ -3759,7 +3759,7 @@ var inputTransactionFormatter = function (options){ options.to = inputAddressFormatter(options.to); } - ['maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { + ['maxFeePerBlobGas', 'maxFeePerGas', 'maxPriorityFeePerGas', 'gasPrice', 'gas', 'value', 'nonce'].filter(function (key) { return options[key] !== undefined; }).forEach(function(key){ options[key] = utils.fromDecimal(options[key]); @@ -3789,6 +3789,9 @@ var outputTransactionFormatter = function (tx){ if(tx.maxPriorityFeePerGas !== undefined) { tx.maxPriorityFeePerGas = utils.toBigNumber(tx.maxPriorityFeePerGas); } + if(tx.maxFeePerBlobGas !== undefined) { + tx.maxFeePerBlobGas = utils.toBigNumber(tx.maxFeePerBlobGas); + } tx.value = utils.toBigNumber(tx.value); return tx; }; @@ -3810,6 +3813,12 @@ var outputTransactionReceiptFormatter = function (receipt){ if(receipt.effectiveGasPrice !== undefined) { receipt.effectiveGasPrice = utils.toBigNumber(receipt.effectiveGasPrice); } + if(receipt.blobGasPrice !== undefined) { + receipt.blobGasPrice = utils.toBigNumber(receipt.blobGasPrice); + } + if(receipt.blobGasUsed !== undefined) { + receipt.blobGasUsed = utils.toBigNumber(receipt.blobGasUsed); + } if(utils.isArray(receipt.logs)) { receipt.logs = receipt.logs.map(function(log){ return outputLogFormatter(log); @@ -3831,11 +3840,17 @@ var outputBlockFormatter = function(block) { if (block.baseFeePerGas !== undefined) { block.baseFeePerGas = utils.toBigNumber(block.baseFeePerGas); } + if (block.blobGasUsed !== undefined) { + block.blobGasUsed = utils.toBigNumber(block.blobGasUsed); + } + if (block.excessBlobGas !== undefined) { + block.excessBlobGas = utils.toBigNumber(block.excessBlobGas); + } block.gasLimit = utils.toDecimal(block.gasLimit); block.gasUsed = utils.toDecimal(block.gasUsed); block.size = utils.toDecimal(block.size); block.timestamp = utils.toDecimal(block.timestamp); - if(block.number !== null) + if (block.number !== null) block.number = utils.toDecimal(block.number); block.difficulty = utils.toBigNumber(block.difficulty); diff --git a/internal/testlog/testlog.go b/internal/testlog/testlog.go index 037b7ee9c120..3cdbea6e0537 100644 --- a/internal/testlog/testlog.go +++ b/internal/testlog/testlog.go @@ -98,6 +98,10 @@ func LoggerWithHandler(t *testing.T, handler slog.Handler) log.Logger { } } +func (l *logger) Handler() slog.Handler { + return l.l.Handler() +} + func (l *logger) Write(level slog.Level, msg string, ctx ...interface{}) {} func (l *logger) Enabled(ctx context.Context, level slog.Level) bool { diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index b86b5909d2cb..1da7d737dd94 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -649,20 +649,6 @@ const MinerJs = ` web3._extend({ property: 'miner', methods: [ - new web3._extend.Method({ - name: 'start', - call: 'miner_start', - }), - new web3._extend.Method({ - name: 'stop', - call: 'miner_stop' - }), - new web3._extend.Method({ - name: 'setEtherbase', - call: 'miner_setEtherbase', - params: 1, - inputFormatter: [web3._extend.formatters.inputAddressFormatter] - }), new web3._extend.Method({ name: 'setExtra', call: 'miner_setExtra', @@ -680,15 +666,6 @@ web3._extend({ params: 1, inputFormatter: [web3._extend.utils.fromDecimal] }), - new web3._extend.Method({ - name: 'setRecommitInterval', - call: 'miner_setRecommitInterval', - params: 1, - }), - new web3._extend.Method({ - name: 'getHashrate', - call: 'miner_getHashrate' - }), ], properties: [] }); diff --git a/log/logger.go b/log/logger.go index 75e364304488..c28bbde56840 100644 --- a/log/logger.go +++ b/log/logger.go @@ -137,6 +137,9 @@ type Logger interface { // Enabled reports whether l emits log records at the given context and level. Enabled(ctx context.Context, level slog.Level) bool + + // Handler returns the underlying handler of the inner logger. + Handler() slog.Handler } type logger struct { @@ -150,6 +153,10 @@ func NewLogger(h slog.Handler) Logger { } } +func (l *logger) Handler() slog.Handler { + return l.inner.Handler() +} + // write logs a message at the specified level: func (l *logger) Write(level slog.Level, msg string, attrs ...any) { if !l.inner.Enabled(context.Background(), level) { diff --git a/log/logger_test.go b/log/logger_test.go index a633f5ad7a4c..ff981fd018ca 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -2,6 +2,7 @@ package log import ( "bytes" + "errors" "fmt" "io" "math/big" @@ -77,7 +78,7 @@ func benchmarkLogger(b *testing.B, l Logger) { tt = time.Now() bigint = big.NewInt(100) nilbig *big.Int - err = fmt.Errorf("Oh nooes it's crap") + err = errors.New("Oh nooes it's crap") ) b.ReportAllocs() b.ResetTimer() @@ -106,7 +107,7 @@ func TestLoggerOutput(t *testing.T) { tt = time.Time{} bigint = big.NewInt(100) nilbig *big.Int - err = fmt.Errorf("Oh nooes it's crap") + err = errors.New("Oh nooes it's crap") smallUint = uint256.NewInt(500_000) bigUint = &uint256.Int{0xff, 0xff, 0xff, 0xff} ) diff --git a/metrics/config.go b/metrics/config.go index 2eb09fb48a33..72f94dd194c9 100644 --- a/metrics/config.go +++ b/metrics/config.go @@ -19,7 +19,7 @@ package metrics // Config contains the configuration for the metric collection. type Config struct { Enabled bool `toml:",omitempty"` - EnabledExpensive bool `toml:",omitempty"` + EnabledExpensive bool `toml:"-"` HTTP string `toml:",omitempty"` Port int `toml:",omitempty"` EnableInfluxDB bool `toml:",omitempty"` diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index bbc4fc024b34..5c8501fd9db5 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -98,20 +98,23 @@ func readMeter(namespace, name string, i interface{}) (string, map[string]interf } return measurement, fields case metrics.ResettingTimer: - t := metric.Snapshot() - if t.Count() == 0 { + ms := metric.Snapshot() + if ms.Count() == 0 { break } - ps := t.Percentiles([]float64{0.50, 0.95, 0.99}) - measurement := fmt.Sprintf("%s%s.span", namespace, name) + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + measurement := fmt.Sprintf("%s%s.timer", namespace, name) fields := map[string]interface{}{ - "count": t.Count(), - "max": t.Max(), - "mean": t.Mean(), - "min": t.Min(), - "p50": int(ps[0]), - "p95": int(ps[1]), - "p99": int(ps[2]), + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], } return measurement, fields } diff --git a/metrics/influxdb/testdata/influxdbv1.want b/metrics/influxdb/testdata/influxdbv1.want index 9443faedc5a2..ded9434c7314 100644 --- a/metrics/influxdb/testdata/influxdbv1.want +++ b/metrics/influxdb/testdata/influxdbv1.want @@ -7,5 +7,5 @@ goth.test/gauge_float64.gauge value=34567.89 978307200000000000 goth.test/gauge_info.gauge value="{\"arch\":\"amd64\",\"commit\":\"7caa2d8163ae3132c1c2d6978c76610caee2d949\",\"os\":\"linux\",\"protocol_versions\":\"64 65 66\",\"version\":\"1.10.18-unstable\"}" 978307200000000000 goth.test/histogram.histogram count=3i,max=3i,mean=2,min=1i,p25=1,p50=2,p75=3,p95=3,p99=3,p999=3,p9999=3,stddev=0.816496580927726,variance=0.6666666666666666 978307200000000000 goth.test/meter.meter count=0i,m1=0,m15=0,m5=0,mean=0 978307200000000000 -goth.test/resetting_timer.span count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000i,p95=120000000i,p99=120000000i 978307200000000000 +goth.test/resetting_timer.timer count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000,p75=40500000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000 978307200000000000 goth.test/timer.timer count=6i,m1=0,m15=0,m5=0,max=120000000i,mean=38333333.333333336,meanrate=0,min=20000000i,p50=22500000,p75=48000000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000,stddev=36545253.529775314,variance=1335555555555555.2 978307200000000000 diff --git a/metrics/influxdb/testdata/influxdbv2.want b/metrics/influxdb/testdata/influxdbv2.want index 9443faedc5a2..ded9434c7314 100644 --- a/metrics/influxdb/testdata/influxdbv2.want +++ b/metrics/influxdb/testdata/influxdbv2.want @@ -7,5 +7,5 @@ goth.test/gauge_float64.gauge value=34567.89 978307200000000000 goth.test/gauge_info.gauge value="{\"arch\":\"amd64\",\"commit\":\"7caa2d8163ae3132c1c2d6978c76610caee2d949\",\"os\":\"linux\",\"protocol_versions\":\"64 65 66\",\"version\":\"1.10.18-unstable\"}" 978307200000000000 goth.test/histogram.histogram count=3i,max=3i,mean=2,min=1i,p25=1,p50=2,p75=3,p95=3,p99=3,p999=3,p9999=3,stddev=0.816496580927726,variance=0.6666666666666666 978307200000000000 goth.test/meter.meter count=0i,m1=0,m15=0,m5=0,mean=0 978307200000000000 -goth.test/resetting_timer.span count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000i,p95=120000000i,p99=120000000i 978307200000000000 +goth.test/resetting_timer.timer count=6i,max=120000000i,mean=30000000,min=10000000i,p50=12500000,p75=40500000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000 978307200000000000 goth.test/timer.timer count=6i,m1=0,m15=0,m5=0,max=120000000i,mean=38333333.333333336,meanrate=0,min=20000000i,p50=22500000,p75=48000000,p95=120000000,p99=120000000,p999=120000000,p9999=120000000,stddev=36545253.529775314,variance=1335555555555555.2 978307200000000000 diff --git a/metrics/metrics.go b/metrics/metrics.go index 9ca8f115c0f7..9e0ac23dd511 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -24,23 +24,12 @@ import ( // for less cluttered pprof profiles. var Enabled = false -// EnabledExpensive is a soft-flag meant for external packages to check if costly -// metrics gathering is allowed or not. The goal is to separate standard metrics -// for health monitoring and debug metrics that might impact runtime performance. -var EnabledExpensive = false - // enablerFlags is the CLI flag names to use to enable metrics collections. var enablerFlags = []string{"metrics"} // enablerEnvVars is the env var names to use to enable metrics collections. var enablerEnvVars = []string{"GETH_METRICS"} -// expensiveEnablerFlags is the CLI flag names to use to enable metrics collections. -var expensiveEnablerFlags = []string{"metrics.expensive"} - -// expensiveEnablerEnvVars is the env var names to use to enable metrics collections. -var expensiveEnablerEnvVars = []string{"GETH_METRICS_EXPENSIVE"} - // Init enables or disables the metrics system. Since we need this to run before // any other code gets to create meters and timers, we'll actually do an ugly hack // and peek into the command line args for the metrics flag. @@ -53,14 +42,6 @@ func init() { } } } - for _, enabler := range expensiveEnablerEnvVars { - if val, found := syscall.Getenv(enabler); found && !EnabledExpensive { - if enable, _ := strconv.ParseBool(val); enable { // ignore error, flag parser will choke on it later - log.Info("Enabling expensive metrics collection") - EnabledExpensive = true - } - } - } for _, arg := range os.Args { flag := strings.TrimLeft(arg, "-") @@ -70,12 +51,6 @@ func init() { Enabled = true } } - for _, enabler := range expensiveEnablerFlags { - if !EnabledExpensive && flag == enabler { - log.Info("Enabling expensive metrics collection") - EnabledExpensive = true - } - } } } diff --git a/metrics/prometheus/collector.go b/metrics/prometheus/collector.go index 25b258d56ab1..353336763b0b 100644 --- a/metrics/prometheus/collector.go +++ b/metrics/prometheus/collector.go @@ -125,12 +125,13 @@ func (c *collector) addResettingTimer(name string, m metrics.ResettingTimerSnaps if m.Count() <= 0 { return } - ps := m.Percentiles([]float64{0.50, 0.95, 0.99}) + pv := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999} + ps := m.Percentiles(pv) c.writeSummaryCounter(name, m.Count()) c.buff.WriteString(fmt.Sprintf(typeSummaryTpl, mutateKey(name))) - c.writeSummaryPercentile(name, "0.50", ps[0]) - c.writeSummaryPercentile(name, "0.95", ps[1]) - c.writeSummaryPercentile(name, "0.99", ps[2]) + for i := range pv { + c.writeSummaryPercentile(name, strconv.FormatFloat(pv[i], 'f', -1, 64), ps[i]) + } c.buff.WriteRune('\n') } diff --git a/metrics/prometheus/testdata/prometheus.want b/metrics/prometheus/testdata/prometheus.want index 861c5f5cf087..a999d83801c6 100644 --- a/metrics/prometheus/testdata/prometheus.want +++ b/metrics/prometheus/testdata/prometheus.want @@ -53,9 +53,12 @@ test_meter 0 test_resetting_timer_count 6 # TYPE test_resetting_timer summary -test_resetting_timer {quantile="0.50"} 1.25e+07 +test_resetting_timer {quantile="0.5"} 1.25e+07 +test_resetting_timer {quantile="0.75"} 4.05e+07 test_resetting_timer {quantile="0.95"} 1.2e+08 test_resetting_timer {quantile="0.99"} 1.2e+08 +test_resetting_timer {quantile="0.999"} 1.2e+08 +test_resetting_timer {quantile="0.9999"} 1.2e+08 # TYPE test_timer_count counter test_timer_count 6 diff --git a/metrics/sample.go b/metrics/sample.go index 5398dd42d5de..bb81e105cf9e 100644 --- a/metrics/sample.go +++ b/metrics/sample.go @@ -148,7 +148,7 @@ func (NilSample) Clear() {} func (NilSample) Snapshot() SampleSnapshot { return (*emptySnapshot)(nil) } func (NilSample) Update(v int64) {} -// SamplePercentiles returns an arbitrary percentile of the slice of int64. +// SamplePercentile returns an arbitrary percentile of the slice of int64. func SamplePercentile(values []int64, p float64) float64 { return CalculatePercentiles(values, []float64{p})[0] } diff --git a/miner/miner.go b/miner/miner.go index 58bb71b557b8..430efcb2fcf1 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -30,9 +30,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) @@ -45,207 +42,124 @@ type Backend interface { // Config is the configuration parameters of mining. type Config struct { - Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards - ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner - GasFloor uint64 // Target gas floor for mined blocks. - GasCeil uint64 // Target gas ceiling for mined blocks. - GasPrice *big.Int // Minimum gas price for mining a transaction - Recommit time.Duration // The time interval for miner to re-create mining work. - - NewPayloadTimeout time.Duration // The maximum time allowance for creating a new payload + Etherbase common.Address `toml:"-"` // Deprecated + PendingFeeRecipient common.Address `toml:"-"` // Address for pending block rewards. + ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner + GasCeil uint64 // Target gas ceiling for mined blocks. + GasPrice *big.Int // Minimum gas price for mining a transaction + Recommit time.Duration // The time interval for miner to re-create mining work. } // DefaultConfig contains default settings for miner. var DefaultConfig = Config{ - GasCeil: 30000000, + GasCeil: 30_000_000, GasPrice: big.NewInt(params.GWei), // The default recommit time is chosen as two seconds since // consensus-layer usually will wait a half slot of time(6s) // for payload generation. It should be enough for Geth to // run 3 rounds. - Recommit: 2 * time.Second, - NewPayloadTimeout: 2 * time.Second, + Recommit: 2 * time.Second, } -// Miner creates blocks and searches for proof-of-work values. +// Miner is the main object which takes care of submitting new work to consensus +// engine and gathering the sealing result. type Miner struct { - mux *event.TypeMux - eth Backend - engine consensus.Engine - exitCh chan struct{} - startCh chan struct{} - stopCh chan struct{} - worker *worker - - wg sync.WaitGroup -} - -func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(header *types.Header) bool) *Miner { - miner := &Miner{ - mux: mux, - eth: eth, - engine: engine, - exitCh: make(chan struct{}), - startCh: make(chan struct{}), - stopCh: make(chan struct{}), - worker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true), - } - miner.wg.Add(1) - go miner.update() - return miner -} - -// update keeps track of the downloader events. Please be aware that this is a one shot type of update loop. -// It's entered once and as soon as `Done` or `Failed` has been broadcasted the events are unregistered and -// the loop is exited. This to prevent a major security vuln where external parties can DOS you with blocks -// and halt your mining operation for as long as the DOS continues. -func (miner *Miner) update() { - defer miner.wg.Done() - - events := miner.mux.Subscribe(downloader.StartEvent{}, downloader.DoneEvent{}, downloader.FailedEvent{}) - defer func() { - if !events.Closed() { - events.Unsubscribe() - } - }() - - shouldStart := false - canStart := true - dlEventCh := events.Chan() - for { - select { - case ev := <-dlEventCh: - if ev == nil { - // Unsubscription done, stop listening - dlEventCh = nil - continue - } - switch ev.Data.(type) { - case downloader.StartEvent: - wasMining := miner.Mining() - miner.worker.stop() - canStart = false - if wasMining { - // Resume mining after sync was finished - shouldStart = true - log.Info("Mining aborted due to sync") - } - miner.worker.syncing.Store(true) - - case downloader.FailedEvent: - canStart = true - if shouldStart { - miner.worker.start() - } - miner.worker.syncing.Store(false) - - case downloader.DoneEvent: - canStart = true - if shouldStart { - miner.worker.start() - } - miner.worker.syncing.Store(false) - - // Stop reacting to downloader events - events.Unsubscribe() - } - case <-miner.startCh: - if canStart { - miner.worker.start() - } - shouldStart = true - case <-miner.stopCh: - shouldStart = false - miner.worker.stop() - case <-miner.exitCh: - miner.worker.close() - return - } + confMu sync.RWMutex // The lock used to protect the config fields: GasCeil, GasTip and Extradata + config *Config + chainConfig *params.ChainConfig + engine consensus.Engine + txpool *txpool.TxPool + chain *core.BlockChain + pending *pending + pendingMu sync.Mutex // Lock protects the pending block +} + +// New creates a new miner with provided config. +func New(eth Backend, config Config, engine consensus.Engine) *Miner { + return &Miner{ + config: &config, + chainConfig: eth.BlockChain().Config(), + engine: engine, + txpool: eth.TxPool(), + chain: eth.BlockChain(), + pending: &pending{}, } } -func (miner *Miner) Start() { - miner.startCh <- struct{}{} -} - -func (miner *Miner) Stop() { - miner.stopCh <- struct{}{} -} - -func (miner *Miner) Close() { - close(miner.exitCh) - miner.wg.Wait() -} - -func (miner *Miner) Mining() bool { - return miner.worker.isRunning() -} - -func (miner *Miner) Hashrate() uint64 { - if pow, ok := miner.engine.(consensus.PoW); ok { - return uint64(pow.Hashrate()) +// Pending returns the currently pending block and associated receipts, logs +// and statedb. The returned values can be nil in case the pending block is +// not initialized. +func (miner *Miner) Pending() (*types.Block, types.Receipts, *state.StateDB) { + pending := miner.getPending() + if pending == nil { + return nil, nil, nil } - return 0 + return pending.block, pending.receipts, pending.stateDB.Copy() } +// SetExtra sets the content used to initialize the block extra field. func (miner *Miner) SetExtra(extra []byte) error { if uint64(len(extra)) > params.MaximumExtraDataSize { return fmt.Errorf("extra exceeds max length. %d > %v", len(extra), params.MaximumExtraDataSize) } - miner.worker.setExtra(extra) + miner.confMu.Lock() + miner.config.ExtraData = extra + miner.confMu.Unlock() return nil } -func (miner *Miner) SetGasTip(tip *big.Int) error { - miner.worker.setGasTip(tip) - return nil -} - -// SetRecommitInterval sets the interval for sealing work resubmitting. -func (miner *Miner) SetRecommitInterval(interval time.Duration) { - miner.worker.setRecommitInterval(interval) -} - -// Pending returns the currently pending block and associated state. The returned -// values can be nil in case the pending block is not initialized -func (miner *Miner) Pending() (*types.Block, *state.StateDB) { - return miner.worker.pending() -} - -// PendingBlock returns the currently pending block. The returned block can be -// nil in case the pending block is not initialized. -// -// Note, to access both the pending block and the pending state -// simultaneously, please use Pending(), as the pending state can -// change between multiple method calls -func (miner *Miner) PendingBlock() *types.Block { - return miner.worker.pendingBlock() -} - -// PendingBlockAndReceipts returns the currently pending block and corresponding receipts. -// The returned values can be nil in case the pending block is not initialized. -func (miner *Miner) PendingBlockAndReceipts() (*types.Block, types.Receipts) { - return miner.worker.pendingBlockAndReceipts() -} - -func (miner *Miner) SetEtherbase(addr common.Address) { - miner.worker.setEtherbase(addr) -} - // SetGasCeil sets the gaslimit to strive for when mining blocks post 1559. // For pre-1559 blocks, it sets the ceiling. func (miner *Miner) SetGasCeil(ceil uint64) { - miner.worker.setGasCeil(ceil) + miner.confMu.Lock() + miner.config.GasCeil = ceil + miner.confMu.Unlock() } -// SubscribePendingLogs starts delivering logs from pending transactions -// to the given channel. -func (miner *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription { - return miner.worker.pendingLogsFeed.Subscribe(ch) +// SetGasTip sets the minimum gas tip for inclusion. +func (miner *Miner) SetGasTip(tip *big.Int) error { + miner.confMu.Lock() + miner.config.GasPrice = tip + miner.confMu.Unlock() + return nil } // BuildPayload builds the payload according to the provided parameters. func (miner *Miner) BuildPayload(args *BuildPayloadArgs) (*Payload, error) { - return miner.worker.buildPayload(args) + return miner.buildPayload(args) +} + +// getPending retrieves the pending block based on the current head block. +// The result might be nil if pending generation is failed. +func (miner *Miner) getPending() *newPayloadResult { + header := miner.chain.CurrentHeader() + miner.pendingMu.Lock() + defer miner.pendingMu.Unlock() + if cached := miner.pending.resolve(header.Hash()); cached != nil { + return cached + } + + var ( + timestamp = uint64(time.Now().Unix()) + withdrawal types.Withdrawals + ) + if miner.chainConfig.IsShanghai(new(big.Int).Add(header.Number, big.NewInt(1)), timestamp) { + withdrawal = []*types.Withdrawal{} + } + ret := miner.generateWork(&generateParams{ + timestamp: timestamp, + forceTime: false, + parentHash: header.Hash(), + coinbase: miner.config.PendingFeeRecipient, + random: common.Hash{}, + withdrawals: withdrawal, + beaconRoot: nil, + noTxs: false, + }) + if ret.err != nil { + return nil + } + miner.pending.update(header.Hash(), ret) + return ret } diff --git a/miner/miner_test.go b/miner/miner_test.go index 5907fb446466..7c39564240c1 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -18,10 +18,9 @@ package miner import ( - "errors" "math/big" + "sync" "testing" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/clique" @@ -33,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" @@ -60,10 +58,6 @@ func (m *mockBackend) TxPool() *txpool.TxPool { return m.txPool } -func (m *mockBackend) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) { - return nil, errors.New("not supported") -} - type testBlockChain struct { root common.Hash config *params.ChainConfig @@ -99,171 +93,18 @@ func (bc *testBlockChain) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) return bc.chainHeadFeed.Subscribe(ch) } -func TestMiner(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - // Subsequent downloader events after a successful DoneEvent should not cause the - // miner to start or stop. This prevents a security vulnerability - // that would allow entities to present fake high blocks that would - // stop mining operations by causing a downloader sync - // until it was discovered they were invalid, whereon mining would resume. - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, true) - - mux.Post(downloader.FailedEvent{}) - waitForMiningState(t, miner, true) -} - -// TestMinerDownloaderFirstFails tests that mining is only -// permitted to run indefinitely once the downloader sees a DoneEvent (success). -// An initial FailedEvent should allow mining to stop on a subsequent -// downloader StartEvent. -func TestMinerDownloaderFirstFails(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.FailedEvent{}) - waitForMiningState(t, miner, true) - - // Since the downloader hasn't yet emitted a successful DoneEvent, - // we expect the miner to stop on next StartEvent. - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - - // Downloader finally succeeds. - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - // Downloader starts again. - // Since it has achieved a DoneEvent once, we expect miner - // state to be unchanged. - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, true) - - mux.Post(downloader.FailedEvent{}) - waitForMiningState(t, miner, true) -} - -func TestMinerStartStopAfterDownloaderEvents(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - - // Downloader finally succeeds. - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - miner.Stop() - waitForMiningState(t, miner, false) - - miner.Start() - waitForMiningState(t, miner, true) - - miner.Stop() - waitForMiningState(t, miner, false) -} - -func TestStartWhileDownload(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - waitForMiningState(t, miner, false) - miner.Start() - waitForMiningState(t, miner, true) - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - // Starting the miner after the downloader should not work - miner.Start() - waitForMiningState(t, miner, false) -} - -func TestStartStopMiner(t *testing.T) { - t.Parallel() - miner, _, cleanup := createMiner(t) - defer cleanup(false) - waitForMiningState(t, miner, false) - miner.Start() - waitForMiningState(t, miner, true) - miner.Stop() - waitForMiningState(t, miner, false) -} - -func TestCloseMiner(t *testing.T) { - t.Parallel() - miner, _, cleanup := createMiner(t) - defer cleanup(true) - waitForMiningState(t, miner, false) - miner.Start() - waitForMiningState(t, miner, true) - // Terminate the miner and wait for the update loop to run - miner.Close() - waitForMiningState(t, miner, false) -} - -// TestMinerSetEtherbase checks that etherbase becomes set even if mining isn't -// possible at the moment -func TestMinerSetEtherbase(t *testing.T) { - t.Parallel() - miner, mux, cleanup := createMiner(t) - defer cleanup(false) - miner.Start() - waitForMiningState(t, miner, true) - // Start the downloader - mux.Post(downloader.StartEvent{}) - waitForMiningState(t, miner, false) - // Now user tries to configure proper mining address - miner.Start() - // Stop the downloader and wait for the update loop to run - mux.Post(downloader.DoneEvent{}) - waitForMiningState(t, miner, true) - - coinbase := common.HexToAddress("0xdeedbeef") - miner.SetEtherbase(coinbase) - if addr := miner.worker.etherbase(); addr != coinbase { - t.Fatalf("Unexpected etherbase want %x got %x", coinbase, addr) - } -} - -// waitForMiningState waits until either -// * the desired mining state was reached -// * a timeout was reached which fails the test -func waitForMiningState(t *testing.T, m *Miner, mining bool) { - t.Helper() - - var state bool - for i := 0; i < 100; i++ { - time.Sleep(10 * time.Millisecond) - if state = m.Mining(); state == mining { - return +func TestBuildPendingBlocks(t *testing.T) { + miner := createMiner(t) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + block, _, _ := miner.Pending() + if block == nil { + t.Error("Pending failed") } - } - t.Fatalf("Mining() == %t, want %t", state, mining) + }() + wg.Wait() } func minerTestGenesisBlock(period uint64, gasLimit uint64, faucet common.Address) *core.Genesis { @@ -294,10 +135,11 @@ func minerTestGenesisBlock(period uint64, gasLimit uint64, faucet common.Address }, } } -func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { + +func createMiner(t *testing.T) *Miner { // Create Ethash config config := Config{ - Etherbase: common.HexToAddress("123456789"), + PendingFeeRecipient: common.HexToAddress("123456789"), } // Create chainConfig chainDB := rawdb.NewMemoryDatabase() @@ -320,18 +162,8 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { pool := legacypool.New(testTxPoolConfig, blockchain) txpool, _ := txpool.New(testTxPoolConfig.PriceLimit, blockchain, []txpool.SubPool{pool}) - backend := NewMockBackend(bc, txpool) - // Create event Mux - mux := new(event.TypeMux) // Create Miner - miner := New(backend, &config, chainConfig, mux, engine, nil) - cleanup := func(skipMiner bool) { - bc.Stop() - engine.Close() - txpool.Close() - if !skipMiner { - miner.Close() - } - } - return miner, mux, cleanup + backend := NewMockBackend(bc, txpool) + miner := New(backend, config, engine) + return miner } diff --git a/miner/ordering.go b/miner/ordering.go index e686656bb2ba..bcf7af46e891 100644 --- a/miner/ordering.go +++ b/miner/ordering.go @@ -21,28 +21,31 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" ) // txWithMinerFee wraps a transaction with its gas price or effective miner gasTipCap type txWithMinerFee struct { tx *txpool.LazyTransaction from common.Address - fees *big.Int + fees *uint256.Int } // newTxWithMinerFee creates a wrapped transaction, calculating the effective // miner gasTipCap if a base fee is provided. // Returns error in case of a negative effective miner gasTipCap. -func newTxWithMinerFee(tx *txpool.LazyTransaction, from common.Address, baseFee *big.Int) (*txWithMinerFee, error) { - tip := new(big.Int).Set(tx.GasTipCap) +func newTxWithMinerFee(tx *txpool.LazyTransaction, from common.Address, baseFee *uint256.Int) (*txWithMinerFee, error) { + tip := new(uint256.Int).Set(tx.GasTipCap) if baseFee != nil { if tx.GasFeeCap.Cmp(baseFee) < 0 { return nil, types.ErrGasFeeCapTooLow } - tip = math.BigMin(tx.GasTipCap, new(big.Int).Sub(tx.GasFeeCap, baseFee)) + tip = new(uint256.Int).Sub(tx.GasFeeCap, baseFee) + if tip.Gt(tx.GasTipCap) { + tip = tx.GasTipCap + } } return &txWithMinerFee{ tx: tx, @@ -87,7 +90,7 @@ type transactionsByPriceAndNonce struct { txs map[common.Address][]*txpool.LazyTransaction // Per account nonce-sorted list of transactions heads txByPriceAndTime // Next transaction for each unique account (price heap) signer types.Signer // Signer for the set of transactions - baseFee *big.Int // Current base fee + baseFee *uint256.Int // Current base fee } // newTransactionsByPriceAndNonce creates a transaction set that can retrieve @@ -96,10 +99,15 @@ type transactionsByPriceAndNonce struct { // Note, the input map is reowned so the caller should not interact any more with // if after providing it to the constructor. func newTransactionsByPriceAndNonce(signer types.Signer, txs map[common.Address][]*txpool.LazyTransaction, baseFee *big.Int) *transactionsByPriceAndNonce { + // Convert the basefee from header format to uint256 format + var baseFeeUint *uint256.Int + if baseFee != nil { + baseFeeUint = uint256.MustFromBig(baseFee) + } // Initialize a price and received time based heap with the head transactions heads := make(txByPriceAndTime, 0, len(txs)) for from, accTxs := range txs { - wrapped, err := newTxWithMinerFee(accTxs[0], from, baseFee) + wrapped, err := newTxWithMinerFee(accTxs[0], from, baseFeeUint) if err != nil { delete(txs, from) continue @@ -114,12 +122,12 @@ func newTransactionsByPriceAndNonce(signer types.Signer, txs map[common.Address] txs: txs, heads: heads, signer: signer, - baseFee: baseFee, + baseFee: baseFeeUint, } } // Peek returns the next transaction by price. -func (t *transactionsByPriceAndNonce) Peek() (*txpool.LazyTransaction, *big.Int) { +func (t *transactionsByPriceAndNonce) Peek() (*txpool.LazyTransaction, *uint256.Int) { if len(t.heads) == 0 { return nil, nil } @@ -145,3 +153,14 @@ func (t *transactionsByPriceAndNonce) Shift() { func (t *transactionsByPriceAndNonce) Pop() { heap.Pop(&t.heads) } + +// Empty returns if the price heap is empty. It can be used to check it simpler +// than calling peek and checking for nil return. +func (t *transactionsByPriceAndNonce) Empty() bool { + return len(t.heads) == 0 +} + +// Clear removes the entire content of the heap. +func (t *transactionsByPriceAndNonce) Clear() { + t.heads, t.txs = nil, nil +} diff --git a/miner/ordering_test.go b/miner/ordering_test.go index d2de9b9f3412..3587a835c884 100644 --- a/miner/ordering_test.go +++ b/miner/ordering_test.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/holiman/uint256" ) func TestTransactionPriceNonceSortLegacy(t *testing.T) { @@ -92,8 +93,8 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) @@ -160,8 +161,8 @@ func TestTransactionTimeSort(t *testing.T) { Hash: tx.Hash(), Tx: tx, Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), + GasFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + GasTipCap: uint256.MustFromBig(tx.GasTipCap()), Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) diff --git a/miner/payload_building.go b/miner/payload_building.go index 719736c4795c..d027cd1e1f3a 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -46,7 +46,6 @@ type BuildPayloadArgs struct { // Id computes an 8-byte identifier by hashing the components of the payload arguments. func (args *BuildPayloadArgs) Id() engine.PayloadID { - // Hash hasher := sha256.New() hasher.Write(args.Parent[:]) binary.Write(hasher, binary.BigEndian, args.Timestamp) @@ -177,7 +176,7 @@ func (payload *Payload) ResolveFull() *engine.ExecutionPayloadEnvelope { } // buildPayload builds the payload according to the provided parameters. -func (w *worker) buildPayload(args *BuildPayloadArgs) (*Payload, error) { +func (miner *Miner) buildPayload(args *BuildPayloadArgs) (*Payload, error) { // Build the initial version with no transaction included. It should be fast // enough to run. The empty payload can at least make sure there is something // to deliver for not missing slot. @@ -191,7 +190,7 @@ func (w *worker) buildPayload(args *BuildPayloadArgs) (*Payload, error) { beaconRoot: args.BeaconRoot, noTxs: true, } - empty := w.getSealingBlock(emptyParams) + empty := miner.generateWork(emptyParams) if empty.err != nil { return nil, empty.err } @@ -227,11 +226,13 @@ func (w *worker) buildPayload(args *BuildPayloadArgs) (*Payload, error) { select { case <-timer.C: start := time.Now() - r := w.getSealingBlock(fullParams) + r := miner.generateWork(fullParams) if r.err == nil { payload.update(r, time.Since(start)) + } else { + log.Info("Error while generating work", "id", payload.id, "err", r.err) } - timer.Reset(w.recommit) + timer.Reset(miner.config.Recommit) case <-payload.stop: log.Info("Stopping work on payload", "id", payload.id, "reason", "delivery") return diff --git a/miner/payload_building_test.go b/miner/payload_building_test.go index 708072b5ecf2..1728b9e5bd59 100644 --- a/miner/payload_building_test.go +++ b/miner/payload_building_test.go @@ -17,26 +17,141 @@ package miner import ( + "math/big" "reflect" "testing" "time" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/beacon/engine" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) +var ( + // Test chain configurations + testTxPoolConfig legacypool.Config + ethashChainConfig *params.ChainConfig + cliqueChainConfig *params.ChainConfig + + // Test accounts + testBankKey, _ = crypto.GenerateKey() + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) + testBankFunds = big.NewInt(1000000000000000000) + + testUserKey, _ = crypto.GenerateKey() + testUserAddress = crypto.PubkeyToAddress(testUserKey.PublicKey) + + // Test transactions + pendingTxs []*types.Transaction + newTxs []*types.Transaction + + testConfig = Config{ + PendingFeeRecipient: testBankAddress, + Recommit: time.Second, + GasCeil: params.GenesisGasLimit, + } +) + +func init() { + testTxPoolConfig = legacypool.DefaultConfig + testTxPoolConfig.Journal = "" + ethashChainConfig = new(params.ChainConfig) + *ethashChainConfig = *params.TestChainConfig + cliqueChainConfig = new(params.ChainConfig) + *cliqueChainConfig = *params.TestChainConfig + cliqueChainConfig.Clique = ¶ms.CliqueConfig{ + Period: 10, + Epoch: 30000, + } + + signer := types.LatestSigner(params.TestChainConfig) + tx1 := types.MustSignNewTx(testBankKey, signer, &types.AccessListTx{ + ChainID: params.TestChainConfig.ChainID, + Nonce: 0, + To: &testUserAddress, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + pendingTxs = append(pendingTxs, tx1) + + tx2 := types.MustSignNewTx(testBankKey, signer, &types.LegacyTx{ + Nonce: 1, + To: &testUserAddress, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + newTxs = append(newTxs, tx2) +} + +// testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing. +type testWorkerBackend struct { + db ethdb.Database + txPool *txpool.TxPool + chain *core.BlockChain + genesis *core.Genesis +} + +func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, n int) *testWorkerBackend { + var gspec = &core.Genesis{ + Config: chainConfig, + Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + } + switch e := engine.(type) { + case *clique.Clique: + gspec.ExtraData = make([]byte, 32+common.AddressLength+crypto.SignatureLength) + copy(gspec.ExtraData[32:32+common.AddressLength], testBankAddress.Bytes()) + e.Authorize(testBankAddress, func(account accounts.Account, s string, data []byte) ([]byte, error) { + return crypto.Sign(crypto.Keccak256(data), testBankKey) + }) + case *ethash.Ethash: + default: + t.Fatalf("unexpected consensus engine type: %T", engine) + } + chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec, nil, engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("core.NewBlockChain failed: %v", err) + } + pool := legacypool.New(testTxPoolConfig, chain) + txpool, _ := txpool.New(testTxPoolConfig.PriceLimit, chain, []txpool.SubPool{pool}) + + return &testWorkerBackend{ + db: db, + chain: chain, + txPool: txpool, + genesis: gspec, + } +} + +func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } +func (b *testWorkerBackend) TxPool() *txpool.TxPool { return b.txPool } + +func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*Miner, *testWorkerBackend) { + backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) + backend.txPool.Add(pendingTxs, true, false) + w := New(backend, testConfig, engine) + return w, backend +} + func TestBuildPayload(t *testing.T) { - t.Parallel() var ( db = rawdb.NewMemoryDatabase() recipient = common.HexToAddress("0xdeadbeef") ) w, b := newTestWorker(t, params.TestChainConfig, ethash.NewFaker(), db, 0) - defer w.close() timestamp := uint64(time.Now().Unix()) args := &BuildPayloadArgs{ diff --git a/miner/pending.go b/miner/pending.go new file mode 100644 index 000000000000..bb91fe89690a --- /dev/null +++ b/miner/pending.go @@ -0,0 +1,67 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package miner + +import ( + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" +) + +// pendingTTL indicates the period of time a generated pending block should +// exist to serve RPC requests before being discarded if the parent block +// has not changed yet. The value is chosen to align with the recommit interval. +const pendingTTL = 2 * time.Second + +// pending wraps a pending block with additional metadata. +type pending struct { + created time.Time + parentHash common.Hash + result *newPayloadResult + lock sync.Mutex +} + +// resolve retrieves the cached pending result if it's available. Nothing will be +// returned if the parentHash is not matched or the result is already too old. +// +// Note, don't modify the returned payload result. +func (p *pending) resolve(parentHash common.Hash) *newPayloadResult { + p.lock.Lock() + defer p.lock.Unlock() + + if p.result == nil { + return nil + } + if parentHash != p.parentHash { + return nil + } + if time.Since(p.created) > pendingTTL { + return nil + } + return p.result +} + +// update refreshes the cached pending block with newly created one. +func (p *pending) update(parent common.Hash, result *newPayloadResult) { + p.lock.Lock() + defer p.lock.Unlock() + + p.parentHash = parent + p.result = result + p.created = time.Now() +} diff --git a/miner/stress/clique/main.go b/miner/stress/clique/main.go deleted file mode 100644 index 60593938458b..000000000000 --- a/miner/stress/clique/main.go +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// This file contains a miner stress test based on the Clique consensus engine. -package main - -import ( - "bytes" - "crypto/ecdsa" - "math/big" - "math/rand" - "os" - "os/signal" - "time" - - "github.com/ethereum/go-ethereum/accounts/keystore" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/fdlimit" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/txpool/legacypool" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/miner" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/params" -) - -func main() { - log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) - fdlimit.Raise(2048) - - // Generate a batch of accounts to seal and fund with - faucets := make([]*ecdsa.PrivateKey, 128) - for i := 0; i < len(faucets); i++ { - faucets[i], _ = crypto.GenerateKey() - } - sealers := make([]*ecdsa.PrivateKey, 4) - for i := 0; i < len(sealers); i++ { - sealers[i], _ = crypto.GenerateKey() - } - // Create a Clique network based off of the Sepolia config - genesis := makeGenesis(faucets, sealers) - - // Handle interrupts. - interruptCh := make(chan os.Signal, 5) - signal.Notify(interruptCh, os.Interrupt) - - var ( - stacks []*node.Node - nodes []*eth.Ethereum - enodes []*enode.Node - ) - for _, sealer := range sealers { - // Start the node and wait until it's up - stack, ethBackend, err := makeSealer(genesis) - if err != nil { - panic(err) - } - defer stack.Close() - - for stack.Server().NodeInfo().Ports.Listener == 0 { - time.Sleep(250 * time.Millisecond) - } - // Connect the node to all the previous ones - for _, n := range enodes { - stack.Server().AddPeer(n) - } - // Start tracking the node and its enode - stacks = append(stacks, stack) - nodes = append(nodes, ethBackend) - enodes = append(enodes, stack.Server().Self()) - - // Inject the signer key and start sealing with it - ks := keystore.NewKeyStore(stack.KeyStoreDir(), keystore.LightScryptN, keystore.LightScryptP) - signer, err := ks.ImportECDSA(sealer, "") - if err != nil { - panic(err) - } - if err := ks.Unlock(signer, ""); err != nil { - panic(err) - } - stack.AccountManager().AddBackend(ks) - } - - // Iterate over all the nodes and start signing on them - time.Sleep(3 * time.Second) - for _, node := range nodes { - if err := node.StartMining(); err != nil { - panic(err) - } - } - time.Sleep(3 * time.Second) - - // Start injecting transactions from the faucet like crazy - nonces := make([]uint64, len(faucets)) - for { - // Stop when interrupted. - select { - case <-interruptCh: - for _, node := range stacks { - node.Close() - } - return - default: - } - - // Pick a random signer node - index := rand.Intn(len(faucets)) - backend := nodes[index%len(nodes)] - - // Create a self transaction and inject into the pool - tx, err := types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000), nil), types.HomesteadSigner{}, faucets[index]) - if err != nil { - panic(err) - } - if err := backend.TxPool().Add([]*types.Transaction{tx}, true, false); err != nil { - panic(err) - } - nonces[index]++ - - // Wait if we're too saturated - if pend, _ := backend.TxPool().Stats(); pend > 2048 { - time.Sleep(100 * time.Millisecond) - } - } -} - -// makeGenesis creates a custom Clique genesis block based on some pre-defined -// signer and faucet accounts. -func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core.Genesis { - // Create a Clique network based off of the Sepolia config - genesis := core.DefaultSepoliaGenesisBlock() - genesis.GasLimit = 25000000 - - genesis.Config.ChainID = big.NewInt(18) - genesis.Config.Clique.Period = 1 - - genesis.Alloc = types.GenesisAlloc{} - for _, faucet := range faucets { - genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = types.Account{ - Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), - } - } - // Sort the signers and embed into the extra-data section - signers := make([]common.Address, len(sealers)) - for i, sealer := range sealers { - signers[i] = crypto.PubkeyToAddress(sealer.PublicKey) - } - for i := 0; i < len(signers); i++ { - for j := i + 1; j < len(signers); j++ { - if bytes.Compare(signers[i][:], signers[j][:]) > 0 { - signers[i], signers[j] = signers[j], signers[i] - } - } - } - genesis.ExtraData = make([]byte, 32+len(signers)*common.AddressLength+65) - for i, signer := range signers { - copy(genesis.ExtraData[32+i*common.AddressLength:], signer[:]) - } - // Return the genesis block for initialization - return genesis -} - -func makeSealer(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { - // Define the basic configurations for the Ethereum node - datadir, _ := os.MkdirTemp("", "") - - config := &node.Config{ - Name: "geth", - Version: params.Version, - DataDir: datadir, - P2P: p2p.Config{ - ListenAddr: "0.0.0.0:0", - NoDiscovery: true, - MaxPeers: 25, - }, - } - // Start the node and configure a full Ethereum node on it - stack, err := node.New(config) - if err != nil { - return nil, nil, err - } - // Create and register the backend - ethBackend, err := eth.New(stack, ðconfig.Config{ - Genesis: genesis, - NetworkId: genesis.Config.ChainID.Uint64(), - SyncMode: downloader.FullSync, - DatabaseCache: 256, - DatabaseHandles: 256, - TxPool: legacypool.DefaultConfig, - GPO: ethconfig.Defaults.GPO, - Miner: miner.Config{ - GasCeil: genesis.GasLimit * 11 / 10, - GasPrice: big.NewInt(1), - Recommit: time.Second, - }, - }) - if err != nil { - return nil, nil, err - } - - err = stack.Start() - return stack, ethBackend, err -} diff --git a/miner/worker.go b/miner/worker.go index 6e4facdd0a94..7e038b0f301b 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -20,12 +20,10 @@ import ( "errors" "fmt" "math/big" - "sync" "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" @@ -33,47 +31,11 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" ) -const ( - // resultQueueSize is the size of channel listening to sealing result. - resultQueueSize = 10 - - // txChanSize is the size of channel listening to NewTxsEvent. - // The number is referenced from the size of tx pool. - txChanSize = 4096 - - // chainHeadChanSize is the size of channel listening to ChainHeadEvent. - chainHeadChanSize = 10 - - // resubmitAdjustChanSize is the size of resubmitting interval adjustment channel. - resubmitAdjustChanSize = 10 - - // minRecommitInterval is the minimal time interval to recreate the sealing block with - // any newly arrived transactions. - minRecommitInterval = 1 * time.Second - - // maxRecommitInterval is the maximum time interval to recreate the sealing block with - // any newly arrived transactions. - maxRecommitInterval = 15 * time.Second - - // intervalAdjustRatio is the impact a single interval adjustment has on sealing work - // resubmitting interval. - intervalAdjustRatio = 0.1 - - // intervalAdjustBias is applied during the new resubmit interval calculation in favor of - // increasing upper limit or decreasing lower limit so that the limit can be reachable. - intervalAdjustBias = 200 * 1000.0 * 1000.0 - - // staleThreshold is the maximum depth of the acceptable stale block. - staleThreshold = 7 -) - var ( errBlockInterruptedByNewHead = errors.New("new head arrived while building block") errBlockInterruptedByRecommit = errors.New("recommit interrupt while building block") @@ -96,47 +58,6 @@ type environment struct { blobs int } -// copy creates a deep copy of environment. -func (env *environment) copy() *environment { - cpy := &environment{ - signer: env.signer, - state: env.state.Copy(), - tcount: env.tcount, - coinbase: env.coinbase, - header: types.CopyHeader(env.header), - receipts: copyReceipts(env.receipts), - } - if env.gasPool != nil { - gasPool := *env.gasPool - cpy.gasPool = &gasPool - } - cpy.txs = make([]*types.Transaction, len(env.txs)) - copy(cpy.txs, env.txs) - - cpy.sidecars = make([]*types.BlobTxSidecar, len(env.sidecars)) - copy(cpy.sidecars, env.sidecars) - - return cpy -} - -// discard terminates the background prefetcher go-routine. It should -// always be called for all created environment instances otherwise -// the go-routine leak can happen. -func (env *environment) discard() { - if env.state == nil { - return - } - env.state.StopPrefetcher() -} - -// task contains all information for consensus engine sealing and result submitting. -type task struct { - receipts []*types.Receipt - state *state.StateDB - block *types.Block - createdAt time.Time -} - const ( commitInterruptNone int32 = iota commitInterruptNewHead @@ -144,627 +65,174 @@ const ( commitInterruptTimeout ) -// newWorkReq represents a request for new sealing work submitting with relative interrupt notifier. -type newWorkReq struct { - interrupt *atomic.Int32 - timestamp int64 -} - // newPayloadResult is the result of payload generation. type newPayloadResult struct { err error block *types.Block fees *big.Int // total block fees sidecars []*types.BlobTxSidecar // collected blobs of blob transactions + stateDB *state.StateDB // StateDB after executing the transactions + receipts []*types.Receipt // Receipts collected during construction } -// getWorkReq represents a request for getting a new sealing work with provided parameters. -type getWorkReq struct { - params *generateParams - result chan *newPayloadResult // non-blocking channel -} - -// intervalAdjust represents a resubmitting interval adjustment. -type intervalAdjust struct { - ratio float64 - inc bool -} - -// worker is the main object which takes care of submitting new work to consensus engine -// and gathering the sealing result. -type worker struct { - config *Config - chainConfig *params.ChainConfig - engine consensus.Engine - eth Backend - chain *core.BlockChain - - // Feeds - pendingLogsFeed event.Feed - - // Subscriptions - mux *event.TypeMux - txsCh chan core.NewTxsEvent - txsSub event.Subscription - chainHeadCh chan core.ChainHeadEvent - chainHeadSub event.Subscription - - // Channels - newWorkCh chan *newWorkReq - getWorkCh chan *getWorkReq - taskCh chan *task - resultCh chan *types.Block - startCh chan struct{} - exitCh chan struct{} - resubmitIntervalCh chan time.Duration - resubmitAdjustCh chan *intervalAdjust - - wg sync.WaitGroup - - current *environment // An environment for current running cycle. - - mu sync.RWMutex // The lock used to protect the coinbase and extra fields - coinbase common.Address - extra []byte - tip *big.Int // Minimum tip needed for non-local transaction to include them - - pendingMu sync.RWMutex - pendingTasks map[common.Hash]*task - - snapshotMu sync.RWMutex // The lock used to protect the snapshots below - snapshotBlock *types.Block - snapshotReceipts types.Receipts - snapshotState *state.StateDB - - // atomic status counters - running atomic.Bool // The indicator whether the consensus engine is running or not. - newTxs atomic.Int32 // New arrival transaction count since last sealing work submitting. - syncing atomic.Bool // The indicator whether the node is still syncing. - - // newpayloadTimeout is the maximum timeout allowance for creating payload. - // The default value is 2 seconds but node operator can set it to arbitrary - // large value. A large timeout allowance may cause Geth to fail creating - // a non-empty payload within the specified time and eventually miss the slot - // in case there are some computation expensive transactions in txpool. - newpayloadTimeout time.Duration - - // recommit is the time interval to re-create sealing work or to re-build - // payload in proof-of-stake stage. - recommit time.Duration - - // External functions - isLocalBlock func(header *types.Header) bool // Function used to determine whether the specified block is mined by local miner. - - // Test hooks - newTaskHook func(*task) // Method to call upon receiving a new sealing task. - skipSealHook func(*task) bool // Method to decide whether skipping the sealing. - fullTaskHook func() // Method to call before pushing the full sealing task. - resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. +// generateParams wraps various of settings for generating sealing task. +type generateParams struct { + timestamp uint64 // The timestamp for sealing task + forceTime bool // Flag whether the given timestamp is immutable or not + parentHash common.Hash // Parent block hash, empty means the latest chain head + coinbase common.Address // The fee recipient address for including transaction + random common.Hash // The randomness generated by beacon chain, empty before the merge + withdrawals types.Withdrawals // List of withdrawals to include in block (shanghai field) + beaconRoot *common.Hash // The beacon root (cancun field). + noTxs bool // Flag whether an empty block without any transaction is expected } -func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(header *types.Header) bool, init bool) *worker { - worker := &worker{ - config: config, - chainConfig: chainConfig, - engine: engine, - eth: eth, - chain: eth.BlockChain(), - mux: mux, - isLocalBlock: isLocalBlock, - coinbase: config.Etherbase, - extra: config.ExtraData, - tip: config.GasPrice, - pendingTasks: make(map[common.Hash]*task), - txsCh: make(chan core.NewTxsEvent, txChanSize), - chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), - newWorkCh: make(chan *newWorkReq), - getWorkCh: make(chan *getWorkReq), - taskCh: make(chan *task), - resultCh: make(chan *types.Block, resultQueueSize), - startCh: make(chan struct{}, 1), - exitCh: make(chan struct{}), - resubmitIntervalCh: make(chan time.Duration), - resubmitAdjustCh: make(chan *intervalAdjust, resubmitAdjustChanSize), - } - // Subscribe for transaction insertion events (whether from network or resurrects) - worker.txsSub = eth.TxPool().SubscribeTransactions(worker.txsCh, true) - // Subscribe events for blockchain - worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) - - // Sanitize recommit interval if the user-specified one is too short. - recommit := worker.config.Recommit - if recommit < minRecommitInterval { - log.Warn("Sanitizing miner recommit interval", "provided", recommit, "updated", minRecommitInterval) - recommit = minRecommitInterval - } - worker.recommit = recommit - - // Sanitize the timeout config for creating payload. - newpayloadTimeout := worker.config.NewPayloadTimeout - if newpayloadTimeout == 0 { - log.Warn("Sanitizing new payload timeout to default", "provided", newpayloadTimeout, "updated", DefaultConfig.NewPayloadTimeout) - newpayloadTimeout = DefaultConfig.NewPayloadTimeout - } - if newpayloadTimeout < time.Millisecond*100 { - log.Warn("Low payload timeout may cause high amount of non-full blocks", "provided", newpayloadTimeout, "default", DefaultConfig.NewPayloadTimeout) +// generateWork generates a sealing block based on the given parameters. +func (miner *Miner) generateWork(params *generateParams) *newPayloadResult { + work, err := miner.prepareWork(params) + if err != nil { + return &newPayloadResult{err: err} } - worker.newpayloadTimeout = newpayloadTimeout - - worker.wg.Add(4) - go worker.mainLoop() - go worker.newWorkLoop(recommit) - go worker.resultLoop() - go worker.taskLoop() + if !params.noTxs { + interrupt := new(atomic.Int32) + timer := time.AfterFunc(miner.config.Recommit, func() { + interrupt.Store(commitInterruptTimeout) + }) + defer timer.Stop() - // Submit first work to initialize pending state. - if init { - worker.startCh <- struct{}{} + err := miner.fillTransactions(interrupt, work) + if errors.Is(err, errBlockInterruptedByTimeout) { + log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(miner.config.Recommit)) + } } - return worker -} - -// setEtherbase sets the etherbase used to initialize the block coinbase field. -func (w *worker) setEtherbase(addr common.Address) { - w.mu.Lock() - defer w.mu.Unlock() - w.coinbase = addr -} - -// etherbase retrieves the configured etherbase address. -func (w *worker) etherbase() common.Address { - w.mu.RLock() - defer w.mu.RUnlock() - return w.coinbase -} - -func (w *worker) setGasCeil(ceil uint64) { - w.mu.Lock() - defer w.mu.Unlock() - w.config.GasCeil = ceil -} - -// setExtra sets the content used to initialize the block extra field. -func (w *worker) setExtra(extra []byte) { - w.mu.Lock() - defer w.mu.Unlock() - w.extra = extra -} - -// setGasTip sets the minimum miner tip needed to include a non-local transaction. -func (w *worker) setGasTip(tip *big.Int) { - w.mu.Lock() - defer w.mu.Unlock() - w.tip = tip -} - -// setRecommitInterval updates the interval for miner sealing work recommitting. -func (w *worker) setRecommitInterval(interval time.Duration) { - select { - case w.resubmitIntervalCh <- interval: - case <-w.exitCh: + block, err := miner.engine.FinalizeAndAssemble(miner.chain, work.header, work.state, work.txs, nil, work.receipts, params.withdrawals) + if err != nil { + return &newPayloadResult{err: err} } -} - -// pending returns the pending state and corresponding block. The returned -// values can be nil in case the pending block is not initialized. -func (w *worker) pending() (*types.Block, *state.StateDB) { - w.snapshotMu.RLock() - defer w.snapshotMu.RUnlock() - if w.snapshotState == nil { - return nil, nil + return &newPayloadResult{ + block: block, + fees: totalFees(block, work.receipts), + sidecars: work.sidecars, + stateDB: work.state, + receipts: work.receipts, } - return w.snapshotBlock, w.snapshotState.Copy() -} - -// pendingBlock returns pending block. The returned block can be nil in case the -// pending block is not initialized. -func (w *worker) pendingBlock() *types.Block { - w.snapshotMu.RLock() - defer w.snapshotMu.RUnlock() - return w.snapshotBlock -} - -// pendingBlockAndReceipts returns pending block and corresponding receipts. -// The returned values can be nil in case the pending block is not initialized. -func (w *worker) pendingBlockAndReceipts() (*types.Block, types.Receipts) { - w.snapshotMu.RLock() - defer w.snapshotMu.RUnlock() - return w.snapshotBlock, w.snapshotReceipts -} - -// start sets the running status as 1 and triggers new work submitting. -func (w *worker) start() { - w.running.Store(true) - w.startCh <- struct{}{} -} - -// stop sets the running status as 0. -func (w *worker) stop() { - w.running.Store(false) } -// isRunning returns an indicator whether worker is running or not. -func (w *worker) isRunning() bool { - return w.running.Load() -} - -// close terminates all background threads maintained by the worker. -// Note the worker does not support being closed multiple times. -func (w *worker) close() { - w.running.Store(false) - close(w.exitCh) - w.wg.Wait() -} +// prepareWork constructs the sealing task according to the given parameters, +// either based on the last chain head or specified parent. In this function +// the pending transactions are not filled yet, only the empty task returned. +func (miner *Miner) prepareWork(genParams *generateParams) (*environment, error) { + miner.confMu.RLock() + defer miner.confMu.RUnlock() -// recalcRecommit recalculates the resubmitting interval upon feedback. -func recalcRecommit(minRecommit, prev time.Duration, target float64, inc bool) time.Duration { - var ( - prevF = float64(prev.Nanoseconds()) - next float64 - ) - if inc { - next = prevF*(1-intervalAdjustRatio) + intervalAdjustRatio*(target+intervalAdjustBias) - max := float64(maxRecommitInterval.Nanoseconds()) - if next > max { - next = max - } - } else { - next = prevF*(1-intervalAdjustRatio) + intervalAdjustRatio*(target-intervalAdjustBias) - min := float64(minRecommit.Nanoseconds()) - if next < min { - next = min + // Find the parent block for sealing task + parent := miner.chain.CurrentBlock() + if genParams.parentHash != (common.Hash{}) { + block := miner.chain.GetBlockByHash(genParams.parentHash) + if block == nil { + return nil, fmt.Errorf("missing parent") } + parent = block.Header() } - return time.Duration(int64(next)) -} - -// newWorkLoop is a standalone goroutine to submit new sealing work upon received events. -func (w *worker) newWorkLoop(recommit time.Duration) { - defer w.wg.Done() - var ( - interrupt *atomic.Int32 - minRecommit = recommit // minimal resubmit interval specified by user. - timestamp int64 // timestamp for each round of sealing. - ) - - timer := time.NewTimer(0) - defer timer.Stop() - <-timer.C // discard the initial tick - - // commit aborts in-flight transaction execution with given signal and resubmits a new one. - commit := func(s int32) { - if interrupt != nil { - interrupt.Store(s) - } - interrupt = new(atomic.Int32) - select { - case w.newWorkCh <- &newWorkReq{interrupt: interrupt, timestamp: timestamp}: - case <-w.exitCh: - return + // Sanity check the timestamp correctness, recap the timestamp + // to parent+1 if the mutation is allowed. + timestamp := genParams.timestamp + if parent.Time >= timestamp { + if genParams.forceTime { + return nil, fmt.Errorf("invalid timestamp, parent %d given %d", parent.Time, timestamp) } - timer.Reset(recommit) - w.newTxs.Store(0) + timestamp = parent.Time + 1 } - // clearPending cleans the stale pending tasks. - clearPending := func(number uint64) { - w.pendingMu.Lock() - for h, t := range w.pendingTasks { - if t.block.NumberU64()+staleThreshold <= number { - delete(w.pendingTasks, h) - } - } - w.pendingMu.Unlock() + // Construct the sealing block header. + header := &types.Header{ + ParentHash: parent.Hash(), + Number: new(big.Int).Add(parent.Number, common.Big1), + GasLimit: core.CalcGasLimit(parent.GasLimit, miner.config.GasCeil), + Time: timestamp, + Coinbase: genParams.coinbase, } - - for { - select { - case <-w.startCh: - clearPending(w.chain.CurrentBlock().Number.Uint64()) - timestamp = time.Now().Unix() - commit(commitInterruptNewHead) - - case head := <-w.chainHeadCh: - clearPending(head.Block.NumberU64()) - timestamp = time.Now().Unix() - commit(commitInterruptNewHead) - - case <-timer.C: - // If sealing is running resubmit a new work cycle periodically to pull in - // higher priced transactions. Disable this overhead for pending blocks. - if w.isRunning() && (w.chainConfig.Clique == nil || w.chainConfig.Clique.Period > 0) { - // Short circuit if no new transaction arrives. - if w.newTxs.Load() == 0 { - timer.Reset(recommit) - continue - } - commit(commitInterruptResubmit) - } - - case interval := <-w.resubmitIntervalCh: - // Adjust resubmit interval explicitly by user. - if interval < minRecommitInterval { - log.Warn("Sanitizing miner recommit interval", "provided", interval, "updated", minRecommitInterval) - interval = minRecommitInterval - } - log.Info("Miner recommit interval update", "from", minRecommit, "to", interval) - minRecommit, recommit = interval, interval - - if w.resubmitHook != nil { - w.resubmitHook(minRecommit, recommit) - } - - case adjust := <-w.resubmitAdjustCh: - // Adjust resubmit interval by feedback. - if adjust.inc { - before := recommit - target := float64(recommit.Nanoseconds()) / adjust.ratio - recommit = recalcRecommit(minRecommit, recommit, target, true) - log.Trace("Increase miner recommit interval", "from", before, "to", recommit) - } else { - before := recommit - recommit = recalcRecommit(minRecommit, recommit, float64(minRecommit.Nanoseconds()), false) - log.Trace("Decrease miner recommit interval", "from", before, "to", recommit) - } - - if w.resubmitHook != nil { - w.resubmitHook(minRecommit, recommit) - } - - case <-w.exitCh: - return - } + // Set the extra field. + if len(miner.config.ExtraData) != 0 { + header.Extra = miner.config.ExtraData } -} - -// mainLoop is responsible for generating and submitting sealing work based on -// the received event. It can support two modes: automatically generate task and -// submit it or return task according to given parameters for various proposes. -func (w *worker) mainLoop() { - defer w.wg.Done() - defer w.txsSub.Unsubscribe() - defer w.chainHeadSub.Unsubscribe() - defer func() { - if w.current != nil { - w.current.discard() - } - }() - - for { - select { - case req := <-w.newWorkCh: - w.commitWork(req.interrupt, req.timestamp) - - case req := <-w.getWorkCh: - req.result <- w.generateWork(req.params) - - case ev := <-w.txsCh: - // Apply transactions to the pending state if we're not sealing - // - // Note all transactions received may not be continuous with transactions - // already included in the current sealing block. These transactions will - // be automatically eliminated. - if !w.isRunning() && w.current != nil { - // If block is already full, abort - if gp := w.current.gasPool; gp != nil && gp.Gas() < params.TxGas { - continue - } - txs := make(map[common.Address][]*txpool.LazyTransaction, len(ev.Txs)) - for _, tx := range ev.Txs { - acc, _ := types.Sender(w.current.signer, tx) - txs[acc] = append(txs[acc], &txpool.LazyTransaction{ - Pool: w.eth.TxPool(), // We don't know where this came from, yolo resolve from everywhere - Hash: tx.Hash(), - Tx: nil, // Do *not* set this! We need to resolve it later to pull blobs in - Time: tx.Time(), - GasFeeCap: tx.GasFeeCap(), - GasTipCap: tx.GasTipCap(), - Gas: tx.Gas(), - BlobGas: tx.BlobGas(), - }) - } - txset := newTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) - tcount := w.current.tcount - w.commitTransactions(w.current, txset, nil, new(big.Int)) - - // Only update the snapshot if any new transactions were added - // to the pending block - if tcount != w.current.tcount { - w.updateSnapshot(w.current) - } - } else { - // Special case, if the consensus engine is 0 period clique(dev mode), - // submit sealing work here since all empty submission will be rejected - // by clique. Of course the advance sealing(empty submission) is disabled. - if w.chainConfig.Clique != nil && w.chainConfig.Clique.Period == 0 { - w.commitWork(nil, time.Now().Unix()) - } - } - w.newTxs.Add(int32(len(ev.Txs))) - - // System stopped - case <-w.exitCh: - return - case <-w.txsSub.Err(): - return - case <-w.chainHeadSub.Err(): - return - } + // Set the randomness field from the beacon chain if it's available. + if genParams.random != (common.Hash{}) { + header.MixDigest = genParams.random } -} - -// taskLoop is a standalone goroutine to fetch sealing task from the generator and -// push them to consensus engine. -func (w *worker) taskLoop() { - defer w.wg.Done() - var ( - stopCh chan struct{} - prev common.Hash - ) - - // interrupt aborts the in-flight sealing task. - interrupt := func() { - if stopCh != nil { - close(stopCh) - stopCh = nil + // Set baseFee and GasLimit if we are on an EIP-1559 chain + if miner.chainConfig.IsLondon(header.Number) { + header.BaseFee = eip1559.CalcBaseFee(miner.chainConfig, parent) + if !miner.chainConfig.IsLondon(parent.Number) { + parentGasLimit := parent.GasLimit * miner.chainConfig.ElasticityMultiplier() + header.GasLimit = core.CalcGasLimit(parentGasLimit, miner.config.GasCeil) } } - for { - select { - case task := <-w.taskCh: - if w.newTaskHook != nil { - w.newTaskHook(task) - } - // Reject duplicate sealing work due to resubmitting. - sealHash := w.engine.SealHash(task.block.Header()) - if sealHash == prev { - continue - } - // Interrupt previous sealing operation - interrupt() - stopCh, prev = make(chan struct{}), sealHash - - if w.skipSealHook != nil && w.skipSealHook(task) { - continue - } - w.pendingMu.Lock() - w.pendingTasks[sealHash] = task - w.pendingMu.Unlock() - - if err := w.engine.Seal(w.chain, task.block, w.resultCh, stopCh); err != nil { - log.Warn("Block sealing failed", "err", err) - w.pendingMu.Lock() - delete(w.pendingTasks, sealHash) - w.pendingMu.Unlock() - } - case <-w.exitCh: - interrupt() - return + // Apply EIP-4844, EIP-4788. + if miner.chainConfig.IsCancun(header.Number, header.Time) { + var excessBlobGas uint64 + if miner.chainConfig.IsCancun(parent.Number, parent.Time) { + excessBlobGas = eip4844.CalcExcessBlobGas(*parent.ExcessBlobGas, *parent.BlobGasUsed) + } else { + // For the first post-fork block, both parent.data_gas_used and parent.excess_data_gas are evaluated as 0 + excessBlobGas = eip4844.CalcExcessBlobGas(0, 0) } + header.BlobGasUsed = new(uint64) + header.ExcessBlobGas = &excessBlobGas + header.ParentBeaconRoot = genParams.beaconRoot } -} - -// resultLoop is a standalone goroutine to handle sealing result submitting -// and flush relative data to the database. -func (w *worker) resultLoop() { - defer w.wg.Done() - for { - select { - case block := <-w.resultCh: - // Short circuit when receiving empty result. - if block == nil { - continue - } - // Short circuit when receiving duplicate result caused by resubmitting. - if w.chain.HasBlock(block.Hash(), block.NumberU64()) { - continue - } - var ( - sealhash = w.engine.SealHash(block.Header()) - hash = block.Hash() - ) - w.pendingMu.RLock() - task, exist := w.pendingTasks[sealhash] - w.pendingMu.RUnlock() - if !exist { - log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", hash) - continue - } - // Different block could share same sealhash, deep copy here to prevent write-write conflict. - var ( - receipts = make([]*types.Receipt, len(task.receipts)) - logs []*types.Log - ) - for i, taskReceipt := range task.receipts { - receipt := new(types.Receipt) - receipts[i] = receipt - *receipt = *taskReceipt - - // add block location fields - receipt.BlockHash = hash - receipt.BlockNumber = block.Number() - receipt.TransactionIndex = uint(i) - - // Update the block hash in all logs since it is now available and not when the - // receipt/log of individual transactions were created. - receipt.Logs = make([]*types.Log, len(taskReceipt.Logs)) - for i, taskLog := range taskReceipt.Logs { - log := new(types.Log) - receipt.Logs[i] = log - *log = *taskLog - log.BlockHash = hash - } - logs = append(logs, receipt.Logs...) - } - // Commit block and state to database. - _, err := w.chain.WriteBlockAndSetHead(block, receipts, logs, task.state, true) - if err != nil { - log.Error("Failed writing block to chain", "err", err) - continue - } - log.Info("Successfully sealed new block", "number", block.Number(), "sealhash", sealhash, "hash", hash, - "elapsed", common.PrettyDuration(time.Since(task.createdAt))) - - // Broadcast the block and announce chain insertion event - w.mux.Post(core.NewMinedBlockEvent{Block: block}) - - case <-w.exitCh: - return - } + // Run the consensus preparation with the default or customized consensus engine. + if err := miner.engine.Prepare(miner.chain, header); err != nil { + log.Error("Failed to prepare header for sealing", "err", err) + return nil, err + } + // Could potentially happen if starting to mine in an odd state. + // Note genParams.coinbase can be different with header.Coinbase + // since clique algorithm can modify the coinbase field in header. + env, err := miner.makeEnv(parent, header, genParams.coinbase) + if err != nil { + log.Error("Failed to create sealing context", "err", err) + return nil, err + } + if header.ParentBeaconRoot != nil { + context := core.NewEVMBlockContext(header, miner.chain, nil) + vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, miner.chainConfig, vm.Config{}) + core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, vmenv, env.state) } + return env, nil } // makeEnv creates a new environment for the sealing block. -func (w *worker) makeEnv(parent *types.Header, header *types.Header, coinbase common.Address) (*environment, error) { +func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase common.Address) (*environment, error) { // Retrieve the parent state to execute on top and start a prefetcher for // the miner to speed block sealing up a bit. - state, err := w.chain.StateAt(parent.Root) + state, err := miner.chain.StateAt(parent.Root) if err != nil { return nil, err } - state.StartPrefetcher("miner") - // Note the passed coinbase may be different with header.Coinbase. - env := &environment{ - signer: types.MakeSigner(w.chainConfig, header.Number, header.Time), + return &environment{ + signer: types.MakeSigner(miner.chainConfig, header.Number, header.Time), state: state, coinbase: coinbase, header: header, - } - // Keep track of transactions which return errors so they can be removed - env.tcount = 0 - return env, nil -} - -// updateSnapshot updates pending snapshot block, receipts and state. -func (w *worker) updateSnapshot(env *environment) { - w.snapshotMu.Lock() - defer w.snapshotMu.Unlock() - - w.snapshotBlock = types.NewBlock( - env.header, - env.txs, - nil, - env.receipts, - trie.NewStackTrie(nil), - ) - w.snapshotReceipts = copyReceipts(env.receipts) - w.snapshotState = env.state.Copy() + }, nil } -func (w *worker) commitTransaction(env *environment, tx *types.Transaction) ([]*types.Log, error) { +func (miner *Miner) commitTransaction(env *environment, tx *types.Transaction) error { if tx.Type() == types.BlobTxType { - return w.commitBlobTransaction(env, tx) + return miner.commitBlobTransaction(env, tx) } - receipt, err := w.applyTransaction(env, tx) + receipt, err := miner.applyTransaction(env, tx) if err != nil { - return nil, err + return err } env.txs = append(env.txs, tx) env.receipts = append(env.receipts, receipt) - return receipt.Logs, nil + env.tcount++ + return nil } -func (w *worker) commitBlobTransaction(env *environment, tx *types.Transaction) ([]*types.Log, error) { +func (miner *Miner) commitBlobTransaction(env *environment, tx *types.Transaction) error { sc := tx.BlobTxSidecar() if sc == nil { panic("blob transaction without blobs in miner") @@ -774,27 +242,28 @@ func (w *worker) commitBlobTransaction(env *environment, tx *types.Transaction) // and not during execution. This means core.ApplyTransaction will not return an error if the // tx has too many blobs. So we have to explicitly check it here. if (env.blobs+len(sc.Blobs))*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { - return nil, errors.New("max data blobs reached") + return errors.New("max data blobs reached") } - receipt, err := w.applyTransaction(env, tx) + receipt, err := miner.applyTransaction(env, tx) if err != nil { - return nil, err + return err } env.txs = append(env.txs, tx.WithoutBlobTxSidecar()) env.receipts = append(env.receipts, receipt) env.sidecars = append(env.sidecars, sc) env.blobs += len(sc.Blobs) *env.header.BlobGasUsed += receipt.BlobGasUsed - return receipt.Logs, nil + env.tcount++ + return nil } // applyTransaction runs the transaction. If execution fails, state and gas pool are reverted. -func (w *worker) applyTransaction(env *environment, tx *types.Transaction) (*types.Receipt, error) { +func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (*types.Receipt, error) { var ( snap = env.state.Snapshot() gp = env.gasPool.Gas() ) - receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &env.coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, *w.chain.GetVMConfig()) + receipt, err := core.ApplyTransaction(miner.chainConfig, miner.chain, &env.coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, *miner.chain.GetVMConfig()) if err != nil { env.state.RevertToSnapshot(snap) env.gasPool.SetGas(gp) @@ -802,13 +271,11 @@ func (w *worker) applyTransaction(env *environment, tx *types.Transaction) (*typ return receipt, err } -func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAndNonce, interrupt *atomic.Int32, minTip *big.Int) error { +func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { gasLimit := env.header.GasLimit if env.gasPool == nil { env.gasPool = new(core.GasPool).AddGas(gasLimit) } - var coalescedLogs []*types.Log - for { // Check interruption signal and abort building if it's fired. if interrupt != nil { @@ -821,8 +288,33 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) break } + // If we don't have enough blob space for any further blob transactions, + // skip that list altogether + if !blobTxs.Empty() && env.blobs*params.BlobTxBlobGasPerBlob >= params.MaxBlobGasPerBlock { + log.Trace("Not enough blob space for further blob transactions") + blobTxs.Clear() + // Fall though to pick up any plain txs + } // Retrieve the next transaction and abort if all done. - ltx, tip := txs.Peek() + var ( + ltx *txpool.LazyTransaction + txs *transactionsByPriceAndNonce + ) + pltx, ptip := plainTxs.Peek() + bltx, btip := blobTxs.Peek() + + switch { + case pltx == nil: + txs, ltx = blobTxs, bltx + case bltx == nil: + txs, ltx = plainTxs, pltx + default: + if ptip.Lt(btip) { + txs, ltx = blobTxs, bltx + } else { + txs, ltx = plainTxs, pltx + } + } if ltx == nil { break } @@ -837,11 +329,6 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn txs.Pop() continue } - // If we don't receive enough tip for the next transaction, skip the account - if tip.Cmp(minTip) < 0 { - log.Trace("Not enough tip for transaction", "hash", ltx.Hash, "tip", tip, "needed", minTip) - break // If the next-best is too low, surely no better will be available - } // Transaction seems to fit, pull it up from the pool tx := ltx.Resolve() if tx == nil { @@ -855,15 +342,15 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn // Check whether the tx is replay protected. If we're not in the EIP155 hf // phase, start ignoring the sender until we do. - if tx.Protected() && !w.chainConfig.IsEIP155(env.header.Number) { - log.Trace("Ignoring replay protected transaction", "hash", ltx.Hash, "eip155", w.chainConfig.EIP155Block) + if tx.Protected() && !miner.chainConfig.IsEIP155(env.header.Number) { + log.Trace("Ignoring replay protected transaction", "hash", ltx.Hash, "eip155", miner.chainConfig.EIP155Block) txs.Pop() continue } // Start executing the transaction env.state.SetTxContext(tx.Hash(), env.tcount) - logs, err := w.commitTransaction(env, tx) + err := miner.commitTransaction(env, tx) switch { case errors.Is(err, core.ErrNonceTooLow): // New head notification data race between the transaction pool and miner, shift @@ -872,8 +359,6 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn case errors.Is(err, nil): // Everything ok, collect the logs and shift in the next transaction from the same account - coalescedLogs = append(coalescedLogs, logs...) - env.tcount++ txs.Shift() default: @@ -883,339 +368,67 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn txs.Pop() } } - if !w.isRunning() && len(coalescedLogs) > 0 { - // We don't push the pendingLogsEvent while we are sealing. The reason is that - // when we are sealing, the worker will regenerate a sealing block every 3 seconds. - // In order to avoid pushing the repeated pendingLog, we disable the pending log pushing. - - // make a copy, the state caches the logs and these logs get "upgraded" from pending to mined - // logs by filling in the block hash when the block was mined by the local miner. This can - // cause a race condition if a log was "upgraded" before the PendingLogsEvent is processed. - cpy := make([]*types.Log, len(coalescedLogs)) - for i, l := range coalescedLogs { - cpy[i] = new(types.Log) - *cpy[i] = *l - } - w.pendingLogsFeed.Send(cpy) - } return nil } -// generateParams wraps various of settings for generating sealing task. -type generateParams struct { - timestamp uint64 // The timestamp for sealing task - forceTime bool // Flag whether the given timestamp is immutable or not - parentHash common.Hash // Parent block hash, empty means the latest chain head - coinbase common.Address // The fee recipient address for including transaction - random common.Hash // The randomness generated by beacon chain, empty before the merge - withdrawals types.Withdrawals // List of withdrawals to include in block. - beaconRoot *common.Hash // The beacon root (cancun field). - noTxs bool // Flag whether an empty block without any transaction is expected -} - -// prepareWork constructs the sealing task according to the given parameters, -// either based on the last chain head or specified parent. In this function -// the pending transactions are not filled yet, only the empty task returned. -func (w *worker) prepareWork(genParams *generateParams) (*environment, error) { - w.mu.RLock() - defer w.mu.RUnlock() - - // Find the parent block for sealing task - parent := w.chain.CurrentBlock() - if genParams.parentHash != (common.Hash{}) { - block := w.chain.GetBlockByHash(genParams.parentHash) - if block == nil { - return nil, fmt.Errorf("missing parent") - } - parent = block.Header() - } - // Sanity check the timestamp correctness, recap the timestamp - // to parent+1 if the mutation is allowed. - timestamp := genParams.timestamp - if parent.Time >= timestamp { - if genParams.forceTime { - return nil, fmt.Errorf("invalid timestamp, parent %d given %d", parent.Time, timestamp) - } - timestamp = parent.Time + 1 - } - // Construct the sealing block header. - header := &types.Header{ - ParentHash: parent.Hash(), - Number: new(big.Int).Add(parent.Number, common.Big1), - GasLimit: core.CalcGasLimit(parent.GasLimit, w.config.GasCeil), - Time: timestamp, - Coinbase: genParams.coinbase, - } - // Set the extra field. - if len(w.extra) != 0 { - header.Extra = w.extra - } - // Set the randomness field from the beacon chain if it's available. - if genParams.random != (common.Hash{}) { - header.MixDigest = genParams.random - } - // Set baseFee and GasLimit if we are on an EIP-1559 chain - if w.chainConfig.IsLondon(header.Number) { - header.BaseFee = eip1559.CalcBaseFee(w.chainConfig, parent) - if !w.chainConfig.IsLondon(parent.Number) { - parentGasLimit := parent.GasLimit * w.chainConfig.ElasticityMultiplier() - header.GasLimit = core.CalcGasLimit(parentGasLimit, w.config.GasCeil) - } - } - // Apply EIP-4844, EIP-4788. - if w.chainConfig.IsCancun(header.Number, header.Time) { - var excessBlobGas uint64 - if w.chainConfig.IsCancun(parent.Number, parent.Time) { - excessBlobGas = eip4844.CalcExcessBlobGas(*parent.ExcessBlobGas, *parent.BlobGasUsed) - } else { - // For the first post-fork block, both parent.data_gas_used and parent.excess_data_gas are evaluated as 0 - excessBlobGas = eip4844.CalcExcessBlobGas(0, 0) - } - header.BlobGasUsed = new(uint64) - header.ExcessBlobGas = &excessBlobGas - header.ParentBeaconRoot = genParams.beaconRoot - } - // Run the consensus preparation with the default or customized consensus engine. - if err := w.engine.Prepare(w.chain, header); err != nil { - log.Error("Failed to prepare header for sealing", "err", err) - return nil, err - } - // Could potentially happen if starting to mine in an odd state. - // Note genParams.coinbase can be different with header.Coinbase - // since clique algorithm can modify the coinbase field in header. - env, err := w.makeEnv(parent, header, genParams.coinbase) - if err != nil { - log.Error("Failed to create sealing context", "err", err) - return nil, err - } - if header.ParentBeaconRoot != nil { - context := core.NewEVMBlockContext(header, w.chain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, w.chainConfig, vm.Config{}) - core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, vmenv, env.state) - } - return env, nil -} - // fillTransactions retrieves the pending transactions from the txpool and fills them // into the given sealing block. The transaction selection and ordering strategy can // be customized with the plugin in the future. -func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) error { - w.mu.RLock() - tip := w.tip - w.mu.RUnlock() +func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) error { + miner.confMu.RLock() + tip := miner.config.GasPrice + miner.confMu.RUnlock() // Retrieve the pending transactions pre-filtered by the 1559/4844 dynamic fees - var baseFee *uint256.Int + filter := txpool.PendingFilter{ + MinTip: uint256.MustFromBig(tip), + } if env.header.BaseFee != nil { - baseFee = uint256.MustFromBig(env.header.BaseFee) + filter.BaseFee = uint256.MustFromBig(env.header.BaseFee) } - var blobFee *uint256.Int if env.header.ExcessBlobGas != nil { - blobFee = uint256.MustFromBig(eip4844.CalcBlobFee(*env.header.ExcessBlobGas)) - } - pending := w.eth.TxPool().Pending(uint256.MustFromBig(tip), baseFee, blobFee) - - // Split the pending transactions into locals and remotes. - localTxs, remoteTxs := make(map[common.Address][]*txpool.LazyTransaction), pending - for _, account := range w.eth.TxPool().Locals() { - if txs := remoteTxs[account]; len(txs) > 0 { - delete(remoteTxs, account) - localTxs[account] = txs - } + filter.BlobFee = uint256.MustFromBig(eip4844.CalcBlobFee(*env.header.ExcessBlobGas)) } + filter.OnlyPlainTxs, filter.OnlyBlobTxs = true, false + pendingPlainTxs := miner.txpool.Pending(filter) - // Fill the block with all available pending transactions. - if len(localTxs) > 0 { - txs := newTransactionsByPriceAndNonce(env.signer, localTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt, new(big.Int)); err != nil { - return err - } - } - if len(remoteTxs) > 0 { - txs := newTransactionsByPriceAndNonce(env.signer, remoteTxs, env.header.BaseFee) - if err := w.commitTransactions(env, txs, interrupt, tip); err != nil { - return err - } - } - return nil -} - -// generateWork generates a sealing block based on the given parameters. -func (w *worker) generateWork(params *generateParams) *newPayloadResult { - work, err := w.prepareWork(params) - if err != nil { - return &newPayloadResult{err: err} - } - defer work.discard() + filter.OnlyPlainTxs, filter.OnlyBlobTxs = false, true + pendingBlobTxs := miner.txpool.Pending(filter) - if !params.noTxs { - interrupt := new(atomic.Int32) - timer := time.AfterFunc(w.newpayloadTimeout, func() { - interrupt.Store(commitInterruptTimeout) - }) - defer timer.Stop() + // Split the pending transactions into locals and remotes. + localPlainTxs, remotePlainTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingPlainTxs + localBlobTxs, remoteBlobTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingBlobTxs - err := w.fillTransactions(interrupt, work) - if errors.Is(err, errBlockInterruptedByTimeout) { - log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(w.newpayloadTimeout)) + for _, account := range miner.txpool.Locals() { + if txs := remotePlainTxs[account]; len(txs) > 0 { + delete(remotePlainTxs, account) + localPlainTxs[account] = txs } - } - block, err := w.engine.FinalizeAndAssemble(w.chain, work.header, work.state, work.txs, nil, work.receipts, params.withdrawals) - if err != nil { - return &newPayloadResult{err: err} - } - return &newPayloadResult{ - block: block, - fees: totalFees(block, work.receipts), - sidecars: work.sidecars, - } -} - -// commitWork generates several new sealing tasks based on the parent block -// and submit them to the sealer. -func (w *worker) commitWork(interrupt *atomic.Int32, timestamp int64) { - // Abort committing if node is still syncing - if w.syncing.Load() { - return - } - start := time.Now() - - // Set the coinbase if the worker is running or it's required - var coinbase common.Address - if w.isRunning() { - coinbase = w.etherbase() - if coinbase == (common.Address{}) { - log.Error("Refusing to mine without etherbase") - return + if txs := remoteBlobTxs[account]; len(txs) > 0 { + delete(remoteBlobTxs, account) + localBlobTxs[account] = txs } } - work, err := w.prepareWork(&generateParams{ - timestamp: uint64(timestamp), - coinbase: coinbase, - }) - if err != nil { - return - } - // Fill pending transactions from the txpool into the block. - err = w.fillTransactions(interrupt, work) - switch { - case err == nil: - // The entire block is filled, decrease resubmit interval in case - // of current interval is larger than the user-specified one. - w.adjustResubmitInterval(&intervalAdjust{inc: false}) + // Fill the block with all available pending transactions. + if len(localPlainTxs) > 0 || len(localBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, localPlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, localBlobTxs, env.header.BaseFee) - case errors.Is(err, errBlockInterruptedByRecommit): - // Notify resubmit loop to increase resubmitting interval if the - // interruption is due to frequent commits. - gaslimit := work.header.GasLimit - ratio := float64(gaslimit-work.gasPool.Gas()) / float64(gaslimit) - if ratio < 0.1 { - ratio = 0.1 + if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { + return err } - w.adjustResubmitInterval(&intervalAdjust{ - ratio: ratio, - inc: true, - }) - - case errors.Is(err, errBlockInterruptedByNewHead): - // If the block building is interrupted by newhead event, discard it - // totally. Committing the interrupted block introduces unnecessary - // delay, and possibly causes miner to mine on the previous head, - // which could result in higher uncle rate. - work.discard() - return - } - // Submit the generated block for consensus sealing. - w.commit(work.copy(), w.fullTaskHook, true, start) - - // Swap out the old work with the new one, terminating any leftover - // prefetcher processes in the mean time and starting a new one. - if w.current != nil { - w.current.discard() } - w.current = work -} + if len(remotePlainTxs) > 0 || len(remoteBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, remotePlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, remoteBlobTxs, env.header.BaseFee) -// commit runs any post-transaction state modifications, assembles the final block -// and commits new work if consensus engine is running. -// Note the assumption is held that the mutation is allowed to the passed env, do -// the deep copy first. -func (w *worker) commit(env *environment, interval func(), update bool, start time.Time) error { - if w.isRunning() { - if interval != nil { - interval() - } - // Create a local environment copy, avoid the data race with snapshot state. - // https://github.com/ethereum/go-ethereum/issues/24299 - env := env.copy() - // Withdrawals are set to nil here, because this is only called in PoW. - block, err := w.engine.FinalizeAndAssemble(w.chain, env.header, env.state, env.txs, nil, env.receipts, nil) - if err != nil { + if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } - // If we're post merge, just ignore - if !w.isTTDReached(block.Header()) { - select { - case w.taskCh <- &task{receipts: env.receipts, state: env.state, block: block, createdAt: time.Now()}: - fees := totalFees(block, env.receipts) - feesInEther := new(big.Float).Quo(new(big.Float).SetInt(fees), big.NewFloat(params.Ether)) - log.Info("Commit new sealing work", "number", block.Number(), "sealhash", w.engine.SealHash(block.Header()), - "txs", env.tcount, "gas", block.GasUsed(), "fees", feesInEther, - "elapsed", common.PrettyDuration(time.Since(start))) - - case <-w.exitCh: - log.Info("Worker has exited") - } - } - } - if update { - w.updateSnapshot(env) } return nil } -// getSealingBlock generates the sealing block based on the given parameters. -// The generation result will be passed back via the given channel no matter -// the generation itself succeeds or not. -func (w *worker) getSealingBlock(params *generateParams) *newPayloadResult { - req := &getWorkReq{ - params: params, - result: make(chan *newPayloadResult, 1), - } - select { - case w.getWorkCh <- req: - return <-req.result - case <-w.exitCh: - return &newPayloadResult{err: errors.New("miner closed")} - } -} - -// isTTDReached returns the indicator if the given block has reached the total -// terminal difficulty for The Merge transition. -func (w *worker) isTTDReached(header *types.Header) bool { - td, ttd := w.chain.GetTd(header.ParentHash, header.Number.Uint64()-1), w.chain.Config().TerminalTotalDifficulty - return td != nil && ttd != nil && td.Cmp(ttd) >= 0 -} - -// adjustResubmitInterval adjusts the resubmit interval. -func (w *worker) adjustResubmitInterval(message *intervalAdjust) { - select { - case w.resubmitAdjustCh <- message: - default: - log.Warn("the resubmitAdjustCh is full, discard the message") - } -} - -// copyReceipts makes a deep copy of the given receipts. -func copyReceipts(receipts []*types.Receipt) []*types.Receipt { - result := make([]*types.Receipt, len(receipts)) - for i, l := range receipts { - cpy := *l - result[i] = &cpy - } - return result -} - // totalFees computes total consumed miner fees in Wei. Block transactions and receipts have to have the same order. func totalFees(block *types.Block, receipts []*types.Receipt) *big.Int { feesWei := new(big.Int) diff --git a/miner/worker_test.go b/miner/worker_test.go deleted file mode 100644 index 9dba12ae51a2..000000000000 --- a/miner/worker_test.go +++ /dev/null @@ -1,510 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package miner - -import ( - "math/big" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/clique" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/txpool" - "github.com/ethereum/go-ethereum/core/txpool/legacypool" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/params" - "github.com/holiman/uint256" -) - -const ( - // testCode is the testing contract binary code which will initialises some - // variables in constructor - testCode = "0x60806040527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0060005534801561003457600080fd5b5060fc806100436000396000f3fe6080604052348015600f57600080fd5b506004361060325760003560e01c80630c4dae8814603757806398a213cf146053575b600080fd5b603d607e565b6040518082815260200191505060405180910390f35b607c60048036036020811015606757600080fd5b81019080803590602001909291905050506084565b005b60005481565b806000819055507fe9e44f9f7da8c559de847a3232b57364adc0354f15a2cd8dc636d54396f9587a6000546040518082815260200191505060405180910390a15056fea265627a7a723058208ae31d9424f2d0bc2a3da1a5dd659db2d71ec322a17db8f87e19e209e3a1ff4a64736f6c634300050a0032" - - // testGas is the gas required for contract deployment. - testGas = 144109 -) - -var ( - // Test chain configurations - testTxPoolConfig legacypool.Config - ethashChainConfig *params.ChainConfig - cliqueChainConfig *params.ChainConfig - - // Test accounts - testBankKey, _ = crypto.GenerateKey() - testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) - testBankFunds = big.NewInt(1000000000000000000) - - testUserKey, _ = crypto.GenerateKey() - testUserAddress = crypto.PubkeyToAddress(testUserKey.PublicKey) - - // Test transactions - pendingTxs []*types.Transaction - newTxs []*types.Transaction - - testConfig = &Config{ - Recommit: time.Second, - GasCeil: params.GenesisGasLimit, - } -) - -func init() { - testTxPoolConfig = legacypool.DefaultConfig - testTxPoolConfig.Journal = "" - ethashChainConfig = new(params.ChainConfig) - *ethashChainConfig = *params.TestChainConfig - cliqueChainConfig = new(params.ChainConfig) - *cliqueChainConfig = *params.TestChainConfig - cliqueChainConfig.Clique = ¶ms.CliqueConfig{ - Period: 10, - Epoch: 30000, - } - - signer := types.LatestSigner(params.TestChainConfig) - tx1 := types.MustSignNewTx(testBankKey, signer, &types.AccessListTx{ - ChainID: params.TestChainConfig.ChainID, - Nonce: 0, - To: &testUserAddress, - Value: big.NewInt(1000), - Gas: params.TxGas, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - pendingTxs = append(pendingTxs, tx1) - - tx2 := types.MustSignNewTx(testBankKey, signer, &types.LegacyTx{ - Nonce: 1, - To: &testUserAddress, - Value: big.NewInt(1000), - Gas: params.TxGas, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - newTxs = append(newTxs, tx2) -} - -// testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing. -type testWorkerBackend struct { - db ethdb.Database - txPool *txpool.TxPool - chain *core.BlockChain - genesis *core.Genesis -} - -func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, n int) *testWorkerBackend { - var gspec = &core.Genesis{ - Config: chainConfig, - Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, - } - switch e := engine.(type) { - case *clique.Clique: - gspec.ExtraData = make([]byte, 32+common.AddressLength+crypto.SignatureLength) - copy(gspec.ExtraData[32:32+common.AddressLength], testBankAddress.Bytes()) - e.Authorize(testBankAddress, func(account accounts.Account, s string, data []byte) ([]byte, error) { - return crypto.Sign(crypto.Keccak256(data), testBankKey) - }) - case *ethash.Ethash: - default: - t.Fatalf("unexpected consensus engine type: %T", engine) - } - chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec, nil, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("core.NewBlockChain failed: %v", err) - } - pool := legacypool.New(testTxPoolConfig, chain) - txpool, _ := txpool.New(testTxPoolConfig.PriceLimit, chain, []txpool.SubPool{pool}) - - return &testWorkerBackend{ - db: db, - chain: chain, - txPool: txpool, - genesis: gspec, - } -} - -func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } -func (b *testWorkerBackend) TxPool() *txpool.TxPool { return b.txPool } - -func (b *testWorkerBackend) newRandomTx(creation bool) *types.Transaction { - var tx *types.Transaction - gasPrice := big.NewInt(10 * params.InitialBaseFee) - if creation { - tx, _ = types.SignTx(types.NewContractCreation(b.txPool.Nonce(testBankAddress), big.NewInt(0), testGas, gasPrice, common.FromHex(testCode)), types.HomesteadSigner{}, testBankKey) - } else { - tx, _ = types.SignTx(types.NewTransaction(b.txPool.Nonce(testBankAddress), testUserAddress, big.NewInt(1000), params.TxGas, gasPrice, nil), types.HomesteadSigner{}, testBankKey) - } - return tx -} - -func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*worker, *testWorkerBackend) { - backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) - backend.txPool.Add(pendingTxs, true, false) - w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false) - w.setEtherbase(testBankAddress) - return w, backend -} - -func TestGenerateAndImportBlock(t *testing.T) { - t.Parallel() - var ( - db = rawdb.NewMemoryDatabase() - config = *params.AllCliqueProtocolChanges - ) - config.Clique = ¶ms.CliqueConfig{Period: 1, Epoch: 30000} - engine := clique.New(config.Clique, db) - - w, b := newTestWorker(t, &config, engine, db, 0) - defer w.close() - - // This test chain imports the mined blocks. - chain, _ := core.NewBlockChain(rawdb.NewMemoryDatabase(), nil, b.genesis, nil, engine, vm.Config{}, nil, nil) - defer chain.Stop() - - // Ignore empty commit here for less noise. - w.skipSealHook = func(task *task) bool { - return len(task.receipts) == 0 - } - - // Wait for mined blocks. - sub := w.mux.Subscribe(core.NewMinedBlockEvent{}) - defer sub.Unsubscribe() - - // Start mining! - w.start() - - for i := 0; i < 5; i++ { - b.txPool.Add([]*types.Transaction{b.newRandomTx(true)}, true, false) - b.txPool.Add([]*types.Transaction{b.newRandomTx(false)}, true, false) - - select { - case ev := <-sub.Chan(): - block := ev.Data.(core.NewMinedBlockEvent).Block - if _, err := chain.InsertChain([]*types.Block{block}); err != nil { - t.Fatalf("failed to insert new mined block %d: %v", block.NumberU64(), err) - } - case <-time.After(3 * time.Second): // Worker needs 1s to include new changes. - t.Fatalf("timeout") - } - } -} - -func TestEmptyWorkEthash(t *testing.T) { - t.Parallel() - testEmptyWork(t, ethashChainConfig, ethash.NewFaker()) -} -func TestEmptyWorkClique(t *testing.T) { - t.Parallel() - testEmptyWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) -} - -func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { - defer engine.Close() - - w, _ := newTestWorker(t, chainConfig, engine, rawdb.NewMemoryDatabase(), 0) - defer w.close() - - taskCh := make(chan struct{}, 2) - checkEqual := func(t *testing.T, task *task) { - // The work should contain 1 tx - receiptLen, balance := 1, uint256.NewInt(1000) - if len(task.receipts) != receiptLen { - t.Fatalf("receipt number mismatch: have %d, want %d", len(task.receipts), receiptLen) - } - if task.state.GetBalance(testUserAddress).Cmp(balance) != 0 { - t.Fatalf("account balance mismatch: have %d, want %d", task.state.GetBalance(testUserAddress), balance) - } - } - w.newTaskHook = func(task *task) { - if task.block.NumberU64() == 1 { - checkEqual(t, task) - taskCh <- struct{}{} - } - } - w.skipSealHook = func(task *task) bool { return true } - w.fullTaskHook = func() { - time.Sleep(100 * time.Millisecond) - } - w.start() // Start mining! - select { - case <-taskCh: - case <-time.NewTimer(3 * time.Second).C: - t.Error("new task timeout") - } -} - -func TestAdjustIntervalEthash(t *testing.T) { - t.Parallel() - testAdjustInterval(t, ethashChainConfig, ethash.NewFaker()) -} - -func TestAdjustIntervalClique(t *testing.T) { - t.Parallel() - testAdjustInterval(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) -} - -func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { - defer engine.Close() - - w, _ := newTestWorker(t, chainConfig, engine, rawdb.NewMemoryDatabase(), 0) - defer w.close() - - w.skipSealHook = func(task *task) bool { - return true - } - w.fullTaskHook = func() { - time.Sleep(100 * time.Millisecond) - } - var ( - progress = make(chan struct{}, 10) - result = make([]float64, 0, 10) - index = 0 - start atomic.Bool - ) - w.resubmitHook = func(minInterval time.Duration, recommitInterval time.Duration) { - // Short circuit if interval checking hasn't started. - if !start.Load() { - return - } - var wantMinInterval, wantRecommitInterval time.Duration - - switch index { - case 0: - wantMinInterval, wantRecommitInterval = 3*time.Second, 3*time.Second - case 1: - origin := float64(3 * time.Second.Nanoseconds()) - estimate := origin*(1-intervalAdjustRatio) + intervalAdjustRatio*(origin/0.8+intervalAdjustBias) - wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(estimate)*time.Nanosecond - case 2: - estimate := result[index-1] - min := float64(3 * time.Second.Nanoseconds()) - estimate = estimate*(1-intervalAdjustRatio) + intervalAdjustRatio*(min-intervalAdjustBias) - wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(estimate)*time.Nanosecond - case 3: - wantMinInterval, wantRecommitInterval = time.Second, time.Second - } - - // Check interval - if minInterval != wantMinInterval { - t.Errorf("resubmit min interval mismatch: have %v, want %v ", minInterval, wantMinInterval) - } - if recommitInterval != wantRecommitInterval { - t.Errorf("resubmit interval mismatch: have %v, want %v", recommitInterval, wantRecommitInterval) - } - result = append(result, float64(recommitInterval.Nanoseconds())) - index += 1 - progress <- struct{}{} - } - w.start() - - time.Sleep(time.Second) // Ensure two tasks have been submitted due to start opt - start.Store(true) - - w.setRecommitInterval(3 * time.Second) - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } - - w.resubmitAdjustCh <- &intervalAdjust{inc: true, ratio: 0.8} - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } - - w.resubmitAdjustCh <- &intervalAdjust{inc: false} - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } - - w.setRecommitInterval(500 * time.Millisecond) - select { - case <-progress: - case <-time.NewTimer(time.Second).C: - t.Error("interval reset timeout") - } -} - -func TestGetSealingWorkEthash(t *testing.T) { - t.Parallel() - testGetSealingWork(t, ethashChainConfig, ethash.NewFaker()) -} - -func TestGetSealingWorkClique(t *testing.T) { - t.Parallel() - testGetSealingWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) -} - -func TestGetSealingWorkPostMerge(t *testing.T) { - t.Parallel() - local := new(params.ChainConfig) - *local = *ethashChainConfig - local.TerminalTotalDifficulty = big.NewInt(0) - testGetSealingWork(t, local, ethash.NewFaker()) -} - -func testGetSealingWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { - defer engine.Close() - - w, b := newTestWorker(t, chainConfig, engine, rawdb.NewMemoryDatabase(), 0) - defer w.close() - - w.setExtra([]byte{0x01, 0x02}) - - w.skipSealHook = func(task *task) bool { - return true - } - w.fullTaskHook = func() { - time.Sleep(100 * time.Millisecond) - } - timestamp := uint64(time.Now().Unix()) - assertBlock := func(block *types.Block, number uint64, coinbase common.Address, random common.Hash) { - if block.Time() != timestamp { - // Sometime the timestamp will be mutated if the timestamp - // is even smaller than parent block's. It's OK. - t.Logf("Invalid timestamp, want %d, get %d", timestamp, block.Time()) - } - _, isClique := engine.(*clique.Clique) - if !isClique { - if len(block.Extra()) != 2 { - t.Error("Unexpected extra field") - } - if block.Coinbase() != coinbase { - t.Errorf("Unexpected coinbase got %x want %x", block.Coinbase(), coinbase) - } - } else { - if block.Coinbase() != (common.Address{}) { - t.Error("Unexpected coinbase") - } - } - if !isClique { - if block.MixDigest() != random { - t.Error("Unexpected mix digest") - } - } - if block.Nonce() != 0 { - t.Error("Unexpected block nonce") - } - if block.NumberU64() != number { - t.Errorf("Mismatched block number, want %d got %d", number, block.NumberU64()) - } - } - var cases = []struct { - parent common.Hash - coinbase common.Address - random common.Hash - expectNumber uint64 - expectErr bool - }{ - { - b.chain.Genesis().Hash(), - common.HexToAddress("0xdeadbeef"), - common.HexToHash("0xcafebabe"), - uint64(1), - false, - }, - { - b.chain.CurrentBlock().Hash(), - common.HexToAddress("0xdeadbeef"), - common.HexToHash("0xcafebabe"), - b.chain.CurrentBlock().Number.Uint64() + 1, - false, - }, - { - b.chain.CurrentBlock().Hash(), - common.Address{}, - common.HexToHash("0xcafebabe"), - b.chain.CurrentBlock().Number.Uint64() + 1, - false, - }, - { - b.chain.CurrentBlock().Hash(), - common.Address{}, - common.Hash{}, - b.chain.CurrentBlock().Number.Uint64() + 1, - false, - }, - { - common.HexToHash("0xdeadbeef"), - common.HexToAddress("0xdeadbeef"), - common.HexToHash("0xcafebabe"), - 0, - true, - }, - } - - // This API should work even when the automatic sealing is not enabled - for _, c := range cases { - r := w.getSealingBlock(&generateParams{ - parentHash: c.parent, - timestamp: timestamp, - coinbase: c.coinbase, - random: c.random, - withdrawals: nil, - beaconRoot: nil, - noTxs: false, - forceTime: true, - }) - if c.expectErr { - if r.err == nil { - t.Error("Expect error but get nil") - } - } else { - if r.err != nil { - t.Errorf("Unexpected error %v", r.err) - } - assertBlock(r.block, c.expectNumber, c.coinbase, c.random) - } - } - - // This API should work even when the automatic sealing is enabled - w.start() - for _, c := range cases { - r := w.getSealingBlock(&generateParams{ - parentHash: c.parent, - timestamp: timestamp, - coinbase: c.coinbase, - random: c.random, - withdrawals: nil, - beaconRoot: nil, - noTxs: false, - forceTime: true, - }) - if c.expectErr { - if r.err == nil { - t.Error("Expect error but get nil") - } - } else { - if r.err != nil { - t.Errorf("Unexpected error %v", r.err) - } - assertBlock(r.block, c.expectNumber, c.coinbase, c.random) - } - } -} diff --git a/node/api.go b/node/api.go index f81f394beb24..a71ae6aa2954 100644 --- a/node/api.go +++ b/node/api.go @@ -145,8 +145,6 @@ func (api *adminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, error) return case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() diff --git a/node/node.go b/node/node.go index dfa83d58c726..c5cb552d2737 100644 --- a/node/node.go +++ b/node/node.go @@ -339,15 +339,9 @@ func (n *Node) closeDataDir() { } } -// obtainJWTSecret loads the jwt-secret, either from the provided config, -// or from the default location. If neither of those are present, it generates -// a new secret and stores to the default location. -func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { - fileName := cliParam - if len(fileName) == 0 { - // no path provided, use default - fileName = n.ResolvePath(datadirJWTKey) - } +// ObtainJWTSecret loads the jwt-secret from the provided config. If the file is not +// present, it generates a new secret and stores to the given location. +func ObtainJWTSecret(fileName string) ([]byte, error) { // try reading from file if data, err := os.ReadFile(fileName); err == nil { jwtSecret := common.FromHex(strings.TrimSpace(string(data))) @@ -373,6 +367,18 @@ func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { return jwtSecret, nil } +// obtainJWTSecret loads the jwt-secret, either from the provided config, +// or from the default location. If neither of those are present, it generates +// a new secret and stores to the default location. +func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { + fileName := cliParam + if len(fileName) == 0 { + // no path provided, use default + fileName = n.ResolvePath(datadirJWTKey) + } + return ObtainJWTSecret(fileName) +} + // startRPC is a helper method to configure all the various RPC endpoints during node // startup. It's not meant to be called at any time afterwards as it makes certain // assumptions about the state of the node. diff --git a/node/node_test.go b/node/node_test.go index 04810a815bf6..d1d1e5dfe8fa 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -415,21 +415,6 @@ func TestRegisterHandler_Successful(t *testing.T) { assert.Equal(t, "success", string(buf)) } -// Tests that the given handler will not be successfully mounted since no HTTP server -// is enabled for RPC -func TestRegisterHandler_Unsuccessful(t *testing.T) { - node, err := New(&DefaultConfig) - if err != nil { - t.Fatalf("could not create new node: %v", err) - } - - // create and mount handler - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("success")) - }) - node.RegisterHandler("test", "/test", handler) -} - // Tests whether websocket requests can be handled on the same port as a regular http server. func TestWebsocketHTTPOnSamePort_WebsocketRequest(t *testing.T) { node := startHTTP(t, 0, 0) diff --git a/node/rpcstack.go b/node/rpcstack.go index d80d5271a7fa..253db0d564a6 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -19,6 +19,7 @@ package node import ( "compress/gzip" "context" + "errors" "fmt" "io" "net" @@ -299,7 +300,7 @@ func (h *httpServer) enableRPC(apis []rpc.API, config httpConfig) error { defer h.mu.Unlock() if h.rpcAllowed() { - return fmt.Errorf("JSON-RPC over HTTP is already enabled") + return errors.New("JSON-RPC over HTTP is already enabled") } // Create RPC server and handler. @@ -335,7 +336,7 @@ func (h *httpServer) enableWS(apis []rpc.API, config wsConfig) error { defer h.mu.Unlock() if h.wsAllowed() { - return fmt.Errorf("JSON-RPC over WebSocket is already enabled") + return errors.New("JSON-RPC over WebSocket is already enabled") } // Create RPC server and handler. srv := rpc.NewServer() diff --git a/p2p/dial.go b/p2p/dial.go index 5e4ab1d50dcc..08e1db28771e 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -25,6 +25,7 @@ import ( mrand "math/rand" "net" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common/mclock" @@ -248,7 +249,7 @@ loop: } case task := <-d.doneCh: - id := task.dest.ID() + id := task.dest().ID() delete(d.dialing, id) d.updateStaticPool(id) d.doneSinceLastLog++ @@ -410,7 +411,7 @@ func (d *dialScheduler) startStaticDials(n int) (started int) { // updateStaticPool attempts to move the given static dial back into staticPool. func (d *dialScheduler) updateStaticPool(id enode.ID) { task, ok := d.static[id] - if ok && task.staticPoolIndex < 0 && d.checkDial(task.dest) == nil { + if ok && task.staticPoolIndex < 0 && d.checkDial(task.dest()) == nil { d.addToStaticPool(task) } } @@ -437,10 +438,11 @@ func (d *dialScheduler) removeFromStaticPool(idx int) { // startDial runs the given dial task in a separate goroutine. func (d *dialScheduler) startDial(task *dialTask) { - d.log.Trace("Starting p2p dial", "id", task.dest.ID(), "ip", task.dest.IP(), "flag", task.flags) - hkey := string(task.dest.ID().Bytes()) + node := task.dest() + d.log.Trace("Starting p2p dial", "id", node.ID(), "ip", node.IP(), "flag", task.flags) + hkey := string(node.ID().Bytes()) d.history.add(hkey, d.clock.Now().Add(dialHistoryExpiration)) - d.dialing[task.dest.ID()] = task + d.dialing[node.ID()] = task go func() { task.run(d) d.doneCh <- task @@ -451,39 +453,46 @@ func (d *dialScheduler) startDial(task *dialTask) { type dialTask struct { staticPoolIndex int flags connFlag + // These fields are private to the task and should not be // accessed by dialScheduler while the task is running. - dest *enode.Node + destPtr atomic.Pointer[enode.Node] lastResolved mclock.AbsTime resolveDelay time.Duration } func newDialTask(dest *enode.Node, flags connFlag) *dialTask { - return &dialTask{dest: dest, flags: flags, staticPoolIndex: -1} + t := &dialTask{flags: flags, staticPoolIndex: -1} + t.destPtr.Store(dest) + return t } type dialError struct { error } +func (t *dialTask) dest() *enode.Node { + return t.destPtr.Load() +} + func (t *dialTask) run(d *dialScheduler) { if t.needResolve() && !t.resolve(d) { return } - err := t.dial(d, t.dest) + err := t.dial(d, t.dest()) if err != nil { // For static nodes, resolve one more time if dialing fails. if _, ok := err.(*dialError); ok && t.flags&staticDialedConn != 0 { if t.resolve(d) { - t.dial(d, t.dest) + t.dial(d, t.dest()) } } } } func (t *dialTask) needResolve() bool { - return t.flags&staticDialedConn != 0 && t.dest.IP() == nil + return t.flags&staticDialedConn != 0 && t.dest().IP() == nil } // resolve attempts to find the current endpoint for the destination @@ -502,29 +511,31 @@ func (t *dialTask) resolve(d *dialScheduler) bool { if t.lastResolved > 0 && time.Duration(d.clock.Now()-t.lastResolved) < t.resolveDelay { return false } - resolved := d.resolver.Resolve(t.dest) + + node := t.dest() + resolved := d.resolver.Resolve(node) t.lastResolved = d.clock.Now() if resolved == nil { t.resolveDelay *= 2 if t.resolveDelay > maxResolveDelay { t.resolveDelay = maxResolveDelay } - d.log.Debug("Resolving node failed", "id", t.dest.ID(), "newdelay", t.resolveDelay) + d.log.Debug("Resolving node failed", "id", node.ID(), "newdelay", t.resolveDelay) return false } // The node was found. t.resolveDelay = initialResolveDelay - t.dest = resolved - d.log.Debug("Resolved node", "id", t.dest.ID(), "addr", &net.TCPAddr{IP: t.dest.IP(), Port: t.dest.TCP()}) + t.destPtr.Store(resolved) + d.log.Debug("Resolved node", "id", resolved.ID(), "addr", &net.TCPAddr{IP: resolved.IP(), Port: resolved.TCP()}) return true } // dial performs the actual connection attempt. func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { dialMeter.Mark(1) - fd, err := d.dialer.Dial(d.ctx, t.dest) + fd, err := d.dialer.Dial(d.ctx, dest) if err != nil { - d.log.Trace("Dial error", "id", t.dest.ID(), "addr", nodeAddr(t.dest), "conn", t.flags, "err", cleanupDialErr(err)) + d.log.Trace("Dial error", "id", dest.ID(), "addr", nodeAddr(dest), "conn", t.flags, "err", cleanupDialErr(err)) dialConnectionError.Mark(1) return &dialError{err} } @@ -532,8 +543,9 @@ func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { } func (t *dialTask) String() string { - id := t.dest.ID() - return fmt.Sprintf("%v %x %v:%d", t.flags, id[:8], t.dest.IP(), t.dest.TCP()) + node := t.dest() + id := node.ID() + return fmt.Sprintf("%v %x %v:%d", t.flags, id[:8], node.IP(), node.TCP()) } func cleanupDialErr(err error) error { diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 988f16b01df2..44b1f5305c45 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -364,7 +364,7 @@ func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { return nil, err } if respN.ID() != n.ID() { - return nil, fmt.Errorf("invalid ID in response record") + return nil, errors.New("invalid ID in response record") } if respN.Seq() < n.Seq() { return n, nil // response record is older diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 8b3e33d37cf7..71f8d8dd0899 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -442,7 +442,7 @@ func (t *UDPv5) verifyResponseNode(c *callV5, r *enr.Record, distances []uint, s } } if _, ok := seen[node.ID()]; ok { - return nil, fmt.Errorf("duplicate record") + return nil, errors.New("duplicate record") } seen[node.ID()] = struct{}{} return node, nil diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 5108910620e0..904a3ddec6f2 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -367,11 +367,11 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey // key is part of the ID nonce signature. var remotePubkey = new(ecdsa.PublicKey) if err := challenge.Node.Load((*enode.Secp256k1)(remotePubkey)); err != nil { - return nil, nil, fmt.Errorf("can't find secp256k1 key for recipient") + return nil, nil, errors.New("can't find secp256k1 key for recipient") } ephkey, err := c.sc.ephemeralKeyGen() if err != nil { - return nil, nil, fmt.Errorf("can't generate ephemeral key") + return nil, nil, errors.New("can't generate ephemeral key") } ephpubkey := EncodePubkey(&ephkey.PublicKey) auth.pubkey = ephpubkey[:] @@ -395,7 +395,7 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey // Create session keys. sec := deriveKeys(sha256.New, ephkey, remotePubkey, c.localnode.ID(), challenge.Node.ID(), cdata) if sec == nil { - return nil, nil, fmt.Errorf("key derivation failed") + return nil, nil, errors.New("key derivation failed") } return auth, sec, err } diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index 8f1c221b8038..4f14d860e1ec 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -191,7 +191,7 @@ func (c *Client) resolveEntry(ctx context.Context, domain, hash string) (entry, func (c *Client) doResolveEntry(ctx context.Context, domain, hash string) (entry, error) { wantHash, err := b32format.DecodeString(hash) if err != nil { - return nil, fmt.Errorf("invalid base32 hash") + return nil, errors.New("invalid base32 hash") } name := hash + "." + domain txts, err := c.cfg.Resolver.LookupTXT(ctx, hash+"."+domain) diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index 7d9703a34558..dfac4fb37208 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -21,6 +21,7 @@ import ( "crypto/ecdsa" "encoding/base32" "encoding/base64" + "errors" "fmt" "io" "strings" @@ -341,7 +342,7 @@ func parseLinkEntry(e string) (entry, error) { func parseLink(e string) (*linkEntry, error) { if !strings.HasPrefix(e, linkPrefix) { - return nil, fmt.Errorf("wrong/missing scheme 'enrtree' in URL") + return nil, errors.New("wrong/missing scheme 'enrtree' in URL") } e = e[len(linkPrefix):] diff --git a/p2p/enode/idscheme.go b/p2p/enode/idscheme.go index fd5d868b761d..6ad7f809a71d 100644 --- a/p2p/enode/idscheme.go +++ b/p2p/enode/idscheme.go @@ -18,7 +18,7 @@ package enode import ( "crypto/ecdsa" - "fmt" + "errors" "io" "github.com/ethereum/go-ethereum/common/math" @@ -67,7 +67,7 @@ func (V4ID) Verify(r *enr.Record, sig []byte) error { if err := r.Load(&entry); err != nil { return err } else if len(entry) != 33 { - return fmt.Errorf("invalid public key") + return errors.New("invalid public key") } h := sha3.NewLegacyKeccak256() diff --git a/p2p/enode/nodedb.go b/p2p/enode/nodedb.go index 7e7fb69b293a..6d55ce17f130 100644 --- a/p2p/enode/nodedb.go +++ b/p2p/enode/nodedb.go @@ -84,7 +84,7 @@ func OpenDB(path string) (*DB, error) { return newPersistentDB(path) } -// newMemoryNodeDB creates a new in-memory node database without a persistent backend. +// newMemoryDB creates a new in-memory node database without a persistent backend. func newMemoryDB() (*DB, error) { db, err := leveldb.Open(storage.NewMemStorage(), nil) if err != nil { @@ -93,7 +93,7 @@ func newMemoryDB() (*DB, error) { return &DB{lvl: db, quit: make(chan struct{})}, nil } -// newPersistentNodeDB creates/opens a leveldb backed persistent node database, +// newPersistentDB creates/opens a leveldb backed persistent node database, // also flushing its contents in case of a version mismatch. func newPersistentDB(path string) (*DB, error) { opts := &opt.Options{OpenFilesCacheCapacity: 5} diff --git a/p2p/nat/natpmp.go b/p2p/nat/natpmp.go index 97601c99dcbb..ea2d8978293f 100644 --- a/p2p/nat/natpmp.go +++ b/p2p/nat/natpmp.go @@ -17,6 +17,7 @@ package nat import ( + "errors" "fmt" "net" "strings" @@ -46,7 +47,7 @@ func (n *pmp) ExternalIP() (net.IP, error) { func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { if lifetime <= 0 { - return 0, fmt.Errorf("lifetime must not be <= 0") + return 0, errors.New("lifetime must not be <= 0") } // Note order of port arguments is switched between our // AddMapping and the client's AddPortMapping. diff --git a/p2p/server.go b/p2p/server.go index 8f42765a8c26..5b7afb4565b9 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -914,13 +914,13 @@ func (srv *Server) checkInboundConn(remoteIP net.IP) error { } // Reject connections that do not match NetRestrict. if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { - return fmt.Errorf("not in netrestrict list") + return errors.New("not in netrestrict list") } // Reject Internet peers that try too often. now := srv.clock.Now() srv.inboundHistory.expire(now, nil) if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { - return fmt.Errorf("too many attempts") + return errors.New("too many attempts") } srv.inboundHistory.add(remoteIP.String(), now.Add(inboundThrottleTime)) return nil @@ -937,7 +937,7 @@ func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) c.transport = srv.newTransport(fd, dialDest.Pubkey()) } - err := srv.setupConn(c, flags, dialDest) + err := srv.setupConn(c, dialDest) if err != nil { if !c.is(inboundConn) { markDialError(err) @@ -947,7 +947,7 @@ func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) return err } -func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) error { +func (srv *Server) setupConn(c *conn, dialDest *enode.Node) error { // Prevent leftover pending conns from entering the handshake. srv.lock.Lock() running := srv.running diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 63cc4936c1bd..17e0f75d5ab9 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -460,7 +460,7 @@ func startExecNodeStack() (*node.Node, error) { // decode the config confEnv := os.Getenv(envNodeConfig) if confEnv == "" { - return nil, fmt.Errorf("missing " + envNodeConfig) + return nil, errors.New("missing " + envNodeConfig) } var conf execNodeConfig if err := json.Unmarshal([]byte(confEnv), &conf); err != nil { diff --git a/p2p/simulations/adapters/inproc_test.go b/p2p/simulations/adapters/inproc_test.go index 2a61508fe18b..d0539ca86752 100644 --- a/p2p/simulations/adapters/inproc_test.go +++ b/p2p/simulations/adapters/inproc_test.go @@ -78,7 +78,7 @@ func TestTCPPipeBidirections(t *testing.T) { } if !bytes.Equal(expected, out) { - t.Fatalf("expected %#v, got %#v", out, expected) + t.Fatalf("expected %#v, got %#v", expected, out) } else { msg := []byte(fmt.Sprintf("pong %02d", i)) if _, err := c2.Write(msg); err != nil { @@ -94,7 +94,7 @@ func TestTCPPipeBidirections(t *testing.T) { t.Fatal(err) } if !bytes.Equal(expected, out) { - t.Fatalf("expected %#v, got %#v", out, expected) + t.Fatalf("expected %#v, got %#v", expected, out) } } } diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index c53a49797bd3..c04308fe0bf8 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -282,8 +282,6 @@ func (t *TestAPI) Events(ctx context.Context) (*rpc.Subscription, error) { return case <-rpcSub.Err(): return - case <-notifier.Closed(): - return } } }() diff --git a/p2p/transport.go b/p2p/transport.go index 4f6bb569bfd3..5fc7686feb06 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -19,6 +19,7 @@ package p2p import ( "bytes" "crypto/ecdsa" + "errors" "fmt" "io" "net" @@ -157,7 +158,7 @@ func readProtocolHandshake(rw MsgReader) (*protoHandshake, error) { return nil, err } if msg.Size > baseProtocolMaxMsgSize { - return nil, fmt.Errorf("message too big") + return nil, errors.New("message too big") } if msg.Code == discMsg { // Disconnect before protocol handshake is valid according to the diff --git a/params/config.go b/params/config.go index 2c80f4f6b09b..439e88218924 100644 --- a/params/config.go +++ b/params/config.go @@ -183,6 +183,7 @@ var ( ArrowGlacierBlock: big.NewInt(0), GrayGlacierBlock: big.NewInt(0), ShanghaiTime: newUint64(0), + CancunTime: newUint64(0), TerminalTotalDifficulty: big.NewInt(0), TerminalTotalDifficultyPassed: true, } @@ -360,6 +361,8 @@ type ChainConfig struct { // TerminalTotalDifficultyPassed is a flag specifying that the network already // passed the terminal total difficulty. Its purpose is to disable legacy sync // even without having seen the TTD locally (safer long term). + // + // TODO(karalabe): Drop this field eventually (always assuming PoS mode) TerminalTotalDifficultyPassed bool `json:"terminalTotalDifficultyPassed,omitempty"` // Various consensus engines @@ -467,7 +470,7 @@ func (c *ChainConfig) Description() string { banner += fmt.Sprintf(" - Shanghai: @%-10v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/shanghai.md)\n", *c.ShanghaiTime) } if c.CancunTime != nil { - banner += fmt.Sprintf(" - Cancun: @%-10v\n", *c.CancunTime) + banner += fmt.Sprintf(" - Cancun: @%-10v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/cancun.md)\n", *c.CancunTime) } if c.PragueTime != nil { banner += fmt.Sprintf(" - Prague: @%-10v\n", *c.PragueTime) @@ -910,6 +913,8 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules if chainID == nil { chainID = new(big.Int) } + // disallow setting Merge out of order + isMerge = isMerge && c.IsLondon(num) return Rules{ ChainID: new(big.Int).Set(chainID), IsHomestead: c.IsHomestead(num), @@ -923,9 +928,9 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules IsBerlin: c.IsBerlin(num), IsLondon: c.IsLondon(num), IsMerge: isMerge, - IsShanghai: c.IsShanghai(num, timestamp), - IsCancun: c.IsCancun(num, timestamp), - IsPrague: c.IsPrague(num, timestamp), - IsVerkle: c.IsVerkle(num, timestamp), + IsShanghai: isMerge && c.IsShanghai(num, timestamp), + IsCancun: isMerge && c.IsCancun(num, timestamp), + IsPrague: isMerge && c.IsPrague(num, timestamp), + IsVerkle: isMerge && c.IsVerkle(num, timestamp), } } diff --git a/params/version.go b/params/version.go index 7284c07524f7..a49385da7d56 100644 --- a/params/version.go +++ b/params/version.go @@ -22,8 +22,8 @@ import ( const ( VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 13 // Patch version component of the current release + VersionMinor = 14 // Minor version component of the current release + VersionPatch = 0 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/rlp/iterator.go b/rlp/iterator.go index 6be574572e61..95bd3f258208 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -23,7 +23,6 @@ type listIterator struct { } // NewListIterator creates an iterator for the (list) represented by data -// TODO: Consider removing this implementation, as it is no longer used. func NewListIterator(data RawValue) (*listIterator, error) { k, t, c, err := readKind(data) if err != nil { diff --git a/rlp/unsafe.go b/rlp/unsafe.go index 2152ba35fc4a..10868caaf287 100644 --- a/rlp/unsafe.go +++ b/rlp/unsafe.go @@ -26,10 +26,5 @@ import ( // byteArrayBytes returns a slice of the byte array v. func byteArrayBytes(v reflect.Value, length int) []byte { - var s []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&s)) - hdr.Data = v.UnsafeAddr() - hdr.Cap = length - hdr.Len = length - return s + return unsafe.Slice((*byte)(unsafe.Pointer(v.UnsafeAddr())), length) } diff --git a/rpc/client.go b/rpc/client.go index 2b0016db8f4e..eef6ee21cf05 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -70,7 +70,7 @@ type BatchElem struct { // discarded. Result interface{} // Error is set if the server returns an error for this request, or if - // unmarshaling into Result fails. It is not set for I/O errors. + // unmarshalling into Result fails. It is not set for I/O errors. Error error } diff --git a/rpc/handler.go b/rpc/handler.go index f44e4d7b01d8..792581cbc0ad 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -324,7 +324,7 @@ func (h *handler) addRequestOp(op *requestOp) { } } -// removeRequestOps stops waiting for the given request IDs. +// removeRequestOp stops waiting for the given request IDs. func (h *handler) removeRequestOp(op *requestOp) { for _, id := range op.ids { delete(h.respWait, string(id)) diff --git a/rpc/http.go b/rpc/http.go index dd376b1ecd59..f4b99429ef4f 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -236,7 +236,7 @@ func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadClos if _, err := buf.ReadFrom(resp.Body); err == nil { body = buf.Bytes() } - + resp.Body.Close() return nil, HTTPError{ Status: resp.Status, StatusCode: resp.StatusCode, diff --git a/rpc/subscription.go b/rpc/subscription.go index 9cb07275479e..d3dff32a272e 100644 --- a/rpc/subscription.go +++ b/rpc/subscription.go @@ -145,12 +145,6 @@ func (n *Notifier) Notify(id ID, data any) error { return nil } -// Closed returns a channel that is closed when the RPC connection is closed. -// Deprecated: use subscription error channel -func (n *Notifier) Closed() <-chan interface{} { - return n.h.conn.closed() -} - // takeSubscription returns the subscription (if one has been created). No subscription can // be created after this call. func (n *Notifier) takeSubscription() *Subscription { diff --git a/rpc/testservice_test.go b/rpc/testservice_test.go index 7d873af6670e..69199e21b711 100644 --- a/rpc/testservice_test.go +++ b/rpc/testservice_test.go @@ -195,10 +195,7 @@ func (s *notificationTestService) SomeSubscription(ctx context.Context, n, val i return } } - select { - case <-notifier.Closed(): - case <-subscription.Err(): - } + <-subscription.Err() if s.unsubscribed != nil { s.unsubscribed <- string(subscription.ID) } diff --git a/rpc/types.go b/rpc/types.go index f88c37c59dad..d12408178615 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -19,6 +19,7 @@ package rpc import ( "context" "encoding/json" + "errors" "fmt" "math" "strings" @@ -104,7 +105,7 @@ func (bn *BlockNumber) UnmarshalJSON(data []byte) error { return err } if blckNum > math.MaxInt64 { - return fmt.Errorf("block number larger than int64") + return errors.New("block number larger than int64") } *bn = BlockNumber(blckNum) return nil @@ -154,7 +155,7 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { err := json.Unmarshal(data, &e) if err == nil { if e.BlockNumber != nil && e.BlockHash != nil { - return fmt.Errorf("cannot specify both BlockHash and BlockNumber, choose one or the other") + return errors.New("cannot specify both BlockHash and BlockNumber, choose one or the other") } bnh.BlockNumber = e.BlockNumber bnh.BlockHash = e.BlockHash @@ -202,7 +203,7 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { return err } if blckNum > math.MaxInt64 { - return fmt.Errorf("blocknumber too high") + return errors.New("blocknumber too high") } bn := BlockNumber(blckNum) bnh.BlockNumber = &bn diff --git a/rpc/types_test.go b/rpc/types_test.go index 617f441d9166..2fa74f9899bb 100644 --- a/rpc/types_test.go +++ b/rpc/types_test.go @@ -45,9 +45,11 @@ func TestBlockNumberJSONUnmarshal(t *testing.T) { 11: {`"pending"`, false, PendingBlockNumber}, 12: {`"latest"`, false, LatestBlockNumber}, 13: {`"earliest"`, false, EarliestBlockNumber}, - 14: {`someString`, true, BlockNumber(0)}, - 15: {`""`, true, BlockNumber(0)}, - 16: {``, true, BlockNumber(0)}, + 14: {`"safe"`, false, SafeBlockNumber}, + 15: {`"finalized"`, false, FinalizedBlockNumber}, + 16: {`someString`, true, BlockNumber(0)}, + 17: {`""`, true, BlockNumber(0)}, + 18: {``, true, BlockNumber(0)}, } for i, test := range tests { @@ -87,18 +89,22 @@ func TestBlockNumberOrHash_UnmarshalJSON(t *testing.T) { 11: {`"pending"`, false, BlockNumberOrHashWithNumber(PendingBlockNumber)}, 12: {`"latest"`, false, BlockNumberOrHashWithNumber(LatestBlockNumber)}, 13: {`"earliest"`, false, BlockNumberOrHashWithNumber(EarliestBlockNumber)}, - 14: {`someString`, true, BlockNumberOrHash{}}, - 15: {`""`, true, BlockNumberOrHash{}}, - 16: {``, true, BlockNumberOrHash{}}, - 17: {`"0x0000000000000000000000000000000000000000000000000000000000000000"`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, - 18: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, - 19: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":false}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, - 20: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":true}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), true)}, - 21: {`{"blockNumber":"0x1"}`, false, BlockNumberOrHashWithNumber(1)}, - 22: {`{"blockNumber":"pending"}`, false, BlockNumberOrHashWithNumber(PendingBlockNumber)}, - 23: {`{"blockNumber":"latest"}`, false, BlockNumberOrHashWithNumber(LatestBlockNumber)}, - 24: {`{"blockNumber":"earliest"}`, false, BlockNumberOrHashWithNumber(EarliestBlockNumber)}, - 25: {`{"blockNumber":"0x1", "blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, true, BlockNumberOrHash{}}, + 14: {`"safe"`, false, BlockNumberOrHashWithNumber(SafeBlockNumber)}, + 15: {`"finalized"`, false, BlockNumberOrHashWithNumber(FinalizedBlockNumber)}, + 16: {`someString`, true, BlockNumberOrHash{}}, + 17: {`""`, true, BlockNumberOrHash{}}, + 18: {``, true, BlockNumberOrHash{}}, + 19: {`"0x0000000000000000000000000000000000000000000000000000000000000000"`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, + 20: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, + 21: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":false}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), false)}, + 22: {`{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","requireCanonical":true}`, false, BlockNumberOrHashWithHash(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), true)}, + 23: {`{"blockNumber":"0x1"}`, false, BlockNumberOrHashWithNumber(1)}, + 24: {`{"blockNumber":"pending"}`, false, BlockNumberOrHashWithNumber(PendingBlockNumber)}, + 25: {`{"blockNumber":"latest"}`, false, BlockNumberOrHashWithNumber(LatestBlockNumber)}, + 26: {`{"blockNumber":"earliest"}`, false, BlockNumberOrHashWithNumber(EarliestBlockNumber)}, + 27: {`{"blockNumber":"safe"}`, false, BlockNumberOrHashWithNumber(SafeBlockNumber)}, + 28: {`{"blockNumber":"finalized"}`, false, BlockNumberOrHashWithNumber(FinalizedBlockNumber)}, + 29: {`{"blockNumber":"0x1", "blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}`, true, BlockNumberOrHash{}}, } for i, test := range tests { @@ -133,6 +139,8 @@ func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { {"pending", int64(PendingBlockNumber)}, {"latest", int64(LatestBlockNumber)}, {"earliest", int64(EarliestBlockNumber)}, + {"safe", int64(SafeBlockNumber)}, + {"finalized", int64(FinalizedBlockNumber)}, } for _, test := range tests { test := test @@ -160,6 +168,8 @@ func TestBlockNumberOrHash_StringAndUnmarshal(t *testing.T) { BlockNumberOrHashWithNumber(PendingBlockNumber), BlockNumberOrHashWithNumber(LatestBlockNumber), BlockNumberOrHashWithNumber(EarliestBlockNumber), + BlockNumberOrHashWithNumber(SafeBlockNumber), + BlockNumberOrHashWithNumber(FinalizedBlockNumber), BlockNumberOrHashWithNumber(32), BlockNumberOrHashWithHash(common.Hash{0xaa}, false), } diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 6bfcd2a727b4..e28f059106f3 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -708,7 +708,7 @@ func formatPrimitiveValue(encType string, encValue interface{}) (string, error) func (t Types) validate() error { for typeKey, typeArr := range t { if len(typeKey) == 0 { - return fmt.Errorf("empty type key") + return errors.New("empty type key") } for i, typeObj := range typeArr { if len(typeObj.Type) == 0 { diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index c6ae7b12743f..f8b3c9d86d1d 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -260,7 +260,7 @@ func fromHex(data any) ([]byte, error) { return nil, fmt.Errorf("wrong type %T", data) } -// typeDataRequest tries to convert the data into a SignDataRequest. +// typedDataRequest tries to convert the data into a SignDataRequest. func typedDataRequest(data any) (*SignDataRequest, error) { var typedData apitypes.TypedData if td, ok := data.(apitypes.TypedData); ok { diff --git a/tests/block_test.go b/tests/block_test.go index fb355085fd8c..1ba84f5f24b6 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -18,7 +18,6 @@ package tests import ( "math/rand" - "runtime" "testing" "github.com/ethereum/go-ethereum/common" @@ -51,9 +50,6 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`.*randomStatetest94.json.*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { - if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 { - t.Skip("test (randomly) skipped on 32-bit windows") - } execBlockTest(t, bt, test) }) // There is also a LegacyTests folder, containing blockchain tests generated @@ -74,20 +70,33 @@ func TestExecutionSpecBlocktests(t *testing.T) { } func execBlockTest(t *testing.T, bt *testMatcher, test *BlockTest) { - if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil, nil)); err != nil { - t.Errorf("test in hash mode without snapshotter failed: %v", err) - return + // If -short flag is used, we don't execute all four permutations, only one. + executionMask := 0xf + if testing.Short() { + executionMask = (1 << (rand.Int63() & 4)) + } + if executionMask&0x1 != 0 { + if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil, nil)); err != nil { + t.Errorf("test in hash mode without snapshotter failed: %v", err) + return + } } - if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil, nil)); err != nil { - t.Errorf("test in hash mode with snapshotter failed: %v", err) - return + if executionMask&0x2 != 0 { + if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil, nil)); err != nil { + t.Errorf("test in hash mode with snapshotter failed: %v", err) + return + } } - if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil, nil)); err != nil { - t.Errorf("test in path mode without snapshotter failed: %v", err) - return + if executionMask&0x4 != 0 { + if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil, nil)); err != nil { + t.Errorf("test in path mode without snapshotter failed: %v", err) + return + } } - if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil, nil)); err != nil { - t.Errorf("test in path mode with snapshotter failed: %v", err) - return + if executionMask&0x8 != 0 { + if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil, nil)); err != nil { + t.Errorf("test in path mode with snapshotter failed: %v", err) + return + } } } diff --git a/tests/state_test.go b/tests/state_test.go index 1d749d8bcf52..6ec5c9d857bc 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -25,7 +25,6 @@ import ( "os" "path/filepath" "reflect" - "runtime" "strings" "testing" "time" @@ -99,15 +98,20 @@ func TestExecutionSpecState(t *testing.T) { } func execStateTest(t *testing.T, st *testMatcher, test *StateTest) { - if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 { - t.Skip("test (randomly) skipped on 32-bit windows") - return - } for _, subtest := range test.Subtests() { subtest := subtest key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) + // If -short flag is used, we don't execute all four permutations, only + // one. + executionMask := 0xf + if testing.Short() { + executionMask = (1 << (rand.Int63() & 4)) + } t.Run(key+"/hash/trie", func(t *testing.T) { + if executionMask&0x1 == 0 { + t.Skip("test (randomly) skipped due to short-tag") + } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { var result error test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, state *StateTestState) { @@ -117,6 +121,9 @@ func execStateTest(t *testing.T, st *testMatcher, test *StateTest) { }) }) t.Run(key+"/hash/snap", func(t *testing.T) { + if executionMask&0x2 == 0 { + t.Skip("test (randomly) skipped due to short-tag") + } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { var result error test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, state *StateTestState) { @@ -132,6 +139,9 @@ func execStateTest(t *testing.T, st *testMatcher, test *StateTest) { }) }) t.Run(key+"/path/trie", func(t *testing.T) { + if executionMask&0x4 == 0 { + t.Skip("test (randomly) skipped due to short-tag") + } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { var result error test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, state *StateTestState) { @@ -141,6 +151,9 @@ func execStateTest(t *testing.T, st *testMatcher, test *StateTest) { }) }) t.Run(key+"/path/snap", func(t *testing.T) { + if executionMask&0x8 == 0 { + t.Skip("test (randomly) skipped due to short-tag") + } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { var result error test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, state *StateTestState) { diff --git a/trie/trie_test.go b/trie/trie_test.go index 379a866f7ea0..920594fdd24f 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -556,7 +556,7 @@ func runRandTest(rt randTest) error { checktr.MustUpdate(it.Key, it.Value) } if tr.Hash() != checktr.Hash() { - rt[i].err = fmt.Errorf("hash mismatch in opItercheckhash") + rt[i].err = errors.New("hash mismatch in opItercheckhash") } case opNodeDiff: var ( @@ -594,19 +594,19 @@ func runRandTest(rt randTest) error { } } if len(insertExp) != len(tr.tracer.inserts) { - rt[i].err = fmt.Errorf("insert set mismatch") + rt[i].err = errors.New("insert set mismatch") } if len(deleteExp) != len(tr.tracer.deletes) { - rt[i].err = fmt.Errorf("delete set mismatch") + rt[i].err = errors.New("delete set mismatch") } for insert := range tr.tracer.inserts { if _, present := insertExp[insert]; !present { - rt[i].err = fmt.Errorf("missing inserted node") + rt[i].err = errors.New("missing inserted node") } } for del := range tr.tracer.deletes { if _, present := deleteExp[del]; !present { - rt[i].err = fmt.Errorf("missing deleted node") + rt[i].err = errors.New("missing deleted node") } } } diff --git a/trie/triestate/state.go b/trie/triestate/state.go index 4c47e9c39734..aa4d32f852f9 100644 --- a/trie/triestate/state.go +++ b/trie/triestate/state.go @@ -59,18 +59,16 @@ type TrieLoader interface { // The value refers to the original content of state before the transition // is made. Nil means that the state was not present previously. type Set struct { - Accounts map[common.Address][]byte // Mutated account set, nil means the account was not present - Storages map[common.Address]map[common.Hash][]byte // Mutated storage set, nil means the slot was not present - Incomplete map[common.Address]struct{} // Indicator whether the storage is incomplete due to large deletion - size common.StorageSize // Approximate size of set + Accounts map[common.Address][]byte // Mutated account set, nil means the account was not present + Storages map[common.Address]map[common.Hash][]byte // Mutated storage set, nil means the slot was not present + size common.StorageSize // Approximate size of set } // New constructs the state set with provided data. -func New(accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte, incomplete map[common.Address]struct{}) *Set { +func New(accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte) *Set { return &Set{ - Accounts: accounts, - Storages: storages, - Incomplete: incomplete, + Accounts: accounts, + Storages: storages, } } @@ -88,7 +86,6 @@ func (s *Set) Size() common.StorageSize { } s.size += common.StorageSize(common.AddressLength) } - s.size += common.StorageSize(common.AddressLength * len(s.Incomplete)) return s.size } diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index f2d6cea635a9..b1e01abac4d6 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -203,7 +203,6 @@ func New(diskdb ethdb.Database, config *Config) *Database { log.Crit("Failed to disable database", "err", err) // impossible to happen } } - log.Warn("Path-based state scheme is an experimental feature") return db } @@ -391,17 +390,20 @@ func (db *Database) Recoverable(root common.Hash) bool { if *id >= dl.stateID() { return false } + // This is a temporary workaround for the unavailability of the freezer in + // dev mode. As a consequence, the Pathdb loses the ability for deep reorg + // in certain cases. + // TODO(rjl493456442): Implement the in-memory ancient store. + if db.freezer == nil { + return false + } // Ensure the requested state is a canonical state and all state // histories in range [id+1, disklayer.ID] are present and complete. - parent := root return checkHistories(db.freezer, *id+1, dl.stateID()-*id, func(m *meta) error { - if m.parent != parent { + if m.parent != root { return errors.New("unexpected state history") } - if len(m.incomplete) > 0 { - return errors.New("incomplete state history") - } - parent = m.root + root = m.root return nil }) == nil } diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index e7bd4699938d..2e7e1bef05ff 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -299,7 +299,7 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode } } } - return root, ctx.nodes, triestate.New(ctx.accountOrigin, ctx.storageOrigin, nil) + return root, ctx.nodes, triestate.New(ctx.accountOrigin, ctx.storageOrigin) } // lastRoot returns the latest root hash, or empty if nothing is cached. @@ -397,7 +397,11 @@ func TestDatabaseRollback(t *testing.T) { if err := tester.db.Recover(parent, loader); err != nil { t.Fatalf("Failed to revert db, err: %v", err) } - tester.verifyState(parent) + if i > 0 { + if err := tester.verifyState(parent); err != nil { + t.Fatalf("Failed to verify state, err: %v", err) + } + } } if tester.db.tree.len() != 1 { t.Fatal("Only disk layer is expected") @@ -539,7 +543,7 @@ func TestCorruptedJournal(t *testing.T) { // Mutate the journal in disk, it should be regarded as invalid blob := rawdb.ReadTrieJournal(tester.db.diskdb) - blob[0] = 1 + blob[0] = 0xa rawdb.WriteTrieJournal(tester.db.diskdb, blob) // Verify states, all not-yet-written states should be discarded diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index ef697cbce8ce..777e4ec8a750 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -17,7 +17,6 @@ package pathdb import ( - "errors" "fmt" "sync" @@ -239,12 +238,6 @@ func (dl *diskLayer) revert(h *history, loader triestate.TrieLoader) (*diskLayer if h.meta.root != dl.rootHash() { return nil, errUnexpectedHistory } - // Reject if the provided state history is incomplete. It's due to - // a large construct SELF-DESTRUCT which can't be handled because - // of memory limitation. - if len(h.meta.incomplete) > 0 { - return nil, errors.New("incomplete state history") - } if dl.id == 0 { return nil, fmt.Errorf("%w: zero state id", errStateUnrecoverable) } diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index 6e3f3faaedce..68fb4809f01d 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -66,7 +66,7 @@ import ( const ( accountIndexSize = common.AddressLength + 13 // The length of encoded account index slotIndexSize = common.HashLength + 5 // The length of encoded slot index - historyMetaSize = 9 + 2*common.HashLength // The length of fixed size part of meta object + historyMetaSize = 9 + 2*common.HashLength // The length of encoded history meta stateHistoryVersion = uint8(0) // initial version of state history structure. ) @@ -192,47 +192,36 @@ func (i *slotIndex) decode(blob []byte) { // meta describes the meta data of state history object. type meta struct { - version uint8 // version tag of history object - parent common.Hash // prev-state root before the state transition - root common.Hash // post-state root after the state transition - block uint64 // associated block number - incomplete []common.Address // list of address whose storage set is incomplete + version uint8 // version tag of history object + parent common.Hash // prev-state root before the state transition + root common.Hash // post-state root after the state transition + block uint64 // associated block number } // encode packs the meta object into byte stream. func (m *meta) encode() []byte { - buf := make([]byte, historyMetaSize+len(m.incomplete)*common.AddressLength) + buf := make([]byte, historyMetaSize) buf[0] = m.version copy(buf[1:1+common.HashLength], m.parent.Bytes()) copy(buf[1+common.HashLength:1+2*common.HashLength], m.root.Bytes()) binary.BigEndian.PutUint64(buf[1+2*common.HashLength:historyMetaSize], m.block) - for i, h := range m.incomplete { - copy(buf[i*common.AddressLength+historyMetaSize:], h.Bytes()) - } return buf[:] } // decode unpacks the meta object from byte stream. func (m *meta) decode(blob []byte) error { if len(blob) < 1 { - return fmt.Errorf("no version tag") + return errors.New("no version tag") } switch blob[0] { case stateHistoryVersion: - if len(blob) < historyMetaSize { + if len(blob) != historyMetaSize { return fmt.Errorf("invalid state history meta, len: %d", len(blob)) } - if (len(blob)-historyMetaSize)%common.AddressLength != 0 { - return fmt.Errorf("corrupted state history meta, len: %d", len(blob)) - } m.version = blob[0] m.parent = common.BytesToHash(blob[1 : 1+common.HashLength]) m.root = common.BytesToHash(blob[1+common.HashLength : 1+2*common.HashLength]) m.block = binary.BigEndian.Uint64(blob[1+2*common.HashLength : historyMetaSize]) - for pos := historyMetaSize; pos < len(blob); { - m.incomplete = append(m.incomplete, common.BytesToAddress(blob[pos:pos+common.AddressLength])) - pos += common.AddressLength - } return nil default: return fmt.Errorf("unknown version %d", blob[0]) @@ -257,7 +246,6 @@ func newHistory(root common.Hash, parent common.Hash, block uint64, states *trie var ( accountList []common.Address storageList = make(map[common.Address][]common.Hash) - incomplete []common.Address ) for addr := range states.Accounts { accountList = append(accountList, addr) @@ -272,18 +260,12 @@ func newHistory(root common.Hash, parent common.Hash, block uint64, states *trie slices.SortFunc(slist, common.Hash.Cmp) storageList[addr] = slist } - for addr := range states.Incomplete { - incomplete = append(incomplete, addr) - } - slices.SortFunc(incomplete, common.Address.Cmp) - return &history{ meta: &meta{ - version: stateHistoryVersion, - parent: parent, - root: root, - block: block, - incomplete: incomplete, + version: stateHistoryVersion, + parent: parent, + root: root, + block: block, }, accounts: states.Accounts, accountList: accountList, diff --git a/triedb/pathdb/history_test.go b/triedb/pathdb/history_test.go index a3257441de80..ab0d44777d7b 100644 --- a/triedb/pathdb/history_test.go +++ b/triedb/pathdb/history_test.go @@ -47,7 +47,7 @@ func randomStateSet(n int) *triestate.Set { account := generateAccount(types.EmptyRootHash) accounts[addr] = types.SlimAccountRLP(account) } - return triestate.New(accounts, storages, nil) + return triestate.New(accounts, storages) } func makeHistory() *history { diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index ac770763e38d..3a0b7ebae273 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -41,7 +41,13 @@ var ( errUnmatchedJournal = errors.New("unmatched journal") ) -const journalVersion uint64 = 0 +// journalVersion ensures that an incompatible journal is detected and discarded. +// +// Changelog: +// +// - Version 0: initial version +// - Version 1: storage.Incomplete field is removed +const journalVersion uint64 = 1 // journalNode represents a trie node persisted in the journal. type journalNode struct { @@ -64,10 +70,9 @@ type journalAccounts struct { // journalStorage represents a list of storage slots belong to an account. type journalStorage struct { - Incomplete bool - Account common.Address - Hashes []common.Hash - Slots [][]byte + Account common.Address + Hashes []common.Hash + Slots [][]byte } // loadJournal tries to parse the layer journal from the disk. @@ -209,11 +214,10 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) { } // Read state changes from journal var ( - jaccounts journalAccounts - jstorages []journalStorage - accounts = make(map[common.Address][]byte) - storages = make(map[common.Address]map[common.Hash][]byte) - incomplete = make(map[common.Address]struct{}) + jaccounts journalAccounts + jstorages []journalStorage + accounts = make(map[common.Address][]byte) + storages = make(map[common.Address]map[common.Hash][]byte) ) if err := r.Decode(&jaccounts); err != nil { return nil, fmt.Errorf("load diff accounts: %v", err) @@ -233,12 +237,9 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) { set[h] = nil } } - if entry.Incomplete { - incomplete[entry.Account] = struct{}{} - } storages[entry.Account] = set } - return db.loadDiffLayer(newDiffLayer(parent, root, parent.stateID()+1, block, nodes, triestate.New(accounts, storages, incomplete)), r) + return db.loadDiffLayer(newDiffLayer(parent, root, parent.stateID()+1, block, nodes, triestate.New(accounts, storages)), r) } // journal implements the layer interface, marshaling the un-flushed trie nodes @@ -316,9 +317,6 @@ func (dl *diffLayer) journal(w io.Writer) error { storage := make([]journalStorage, 0, len(dl.states.Storages)) for addr, slots := range dl.states.Storages { entry := journalStorage{Account: addr} - if _, ok := dl.states.Incomplete[addr]; ok { - entry.Incomplete = true - } for slotHash, slot := range slots { entry.Hashes = append(entry.Hashes, slotHash) entry.Slots = append(entry.Slots, slot) diff --git a/triedb/pathdb/nodebuffer.go b/triedb/pathdb/nodebuffer.go index 4a7d328b9afb..8f84c2b44207 100644 --- a/triedb/pathdb/nodebuffer.go +++ b/triedb/pathdb/nodebuffer.go @@ -204,6 +204,19 @@ func (b *nodebuffer) setSize(size int, db ethdb.KeyValueStore, clean *fastcache. return b.flush(db, clean, id, false) } +// allocBatch returns a database batch with pre-allocated buffer. +func (b *nodebuffer) allocBatch(db ethdb.KeyValueStore) ethdb.Batch { + var metasize int + for owner, nodes := range b.nodes { + if owner == (common.Hash{}) { + metasize += len(nodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix + } else { + metasize += len(nodes) * (len(rawdb.TrieNodeStoragePrefix) + common.HashLength) // database key prefix + owner + } + } + return db.NewBatchWithSize((metasize + int(b.size)) * 11 / 10) // extra 10% for potential pebble internal stuff +} + // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error { @@ -217,7 +230,7 @@ func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id ui } var ( start = time.Now() - batch = db.NewBatchWithSize(int(b.size)) + batch = b.allocBatch(db) ) nodes := writeNodes(batch, b.nodes, clean) rawdb.WritePersistentStateID(batch, id)