Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: TxDAG generation v0.1 version #187

Open
wants to merge 42 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 41 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
a6ce37c
txdag: the initial the TxDAG commit;
galaio Jul 30, 2024
22d858d
txdag: opt txdag encoding;
galaio Aug 2, 2024
71f4525
mvstates: fix oom issue when mining is enabled;
galaio Aug 23, 2024
a70fe85
txDAG transfer (#28)
andyzhang2023 Aug 13, 2024
5f9c619
fix bug: only generate DAG tx after both 'remote' and 'local' transac…
Aug 27, 2024
560af71
fix bug: only generate DAG tx after both 'remote' and 'local' transac…
Aug 27, 2024
63c802a
fix: append dag tx to the end of bundle txs
Aug 29, 2024
7ef7ba8
fix: copy with mv state
Aug 29, 2024
d11a90b
mvstates: abandon RWKey, using origin key to record rwset;
galaio Aug 29, 2024
38040aa
add gaslimit reservation
Aug 30, 2024
9eb6f8f
fix: estimate the gas for TxDAG transaction
Aug 30, 2024
7332f06
fix: correctly estimate the max gaslimit for TxDAG data
Sep 5, 2024
0001951
txdag: add metrics when mining;
galaio Sep 3, 2024
4b57be3
txdag: fix mvstates copy issue;
galaio Sep 4, 2024
69c6c71
txdag: add timeout for async generation;
galaio Sep 4, 2024
e74d13d
txdag: fix nil pointer issue;
galaio Sep 4, 2024
60e4cbf
txdag: fix mining reset tx issue;
galaio Sep 5, 2024
0593ec3
txdag: add debug log;
galaio Sep 5, 2024
9c9e329
txdag: fix txdep cache logic to replace failed tx;
galaio Sep 5, 2024
1ae3839
txdag: clean code, abandon useless codes, add more async logic;
galaio Aug 29, 2024
39100e7
txdag: adaptor txdag for mining;
galaio Sep 4, 2024
e483990
txdag: clean codes;
galaio Sep 4, 2024
80b1c5a
txdag: opt generation logic, support stop mvstates;
galaio Sep 5, 2024
44a78b8
txdag: fix record panic after stop the async gen;
galaio Sep 6, 2024
31263e7
txdag: fix the last tx dep wrong when mining;
galaio Sep 6, 2024
778fa73
txdag: opt 100% conflict scenario perf, reduce send chan frequency;
galaio Sep 6, 2024
1b3b498
txdag: opt rwset string format;
galaio Sep 6, 2024
d0526cc
txdag: opt rwset string format;
galaio Sep 7, 2024
bf69635
txdag: reduce more mem usage;
galaio Sep 10, 2024
051107e
txdag: using a new mem pool;
galaio Sep 12, 2024
e0d56ca
txdag: using a new mem pool;
galaio Sep 12, 2024
f65ca0a
txdag: clean codes;
galaio Sep 23, 2024
e9b1e2e
txdag: support find all prev tx from rw list;
galaio Sep 24, 2024
917aaa0
txdag: support find all prev tx from rw list;
galaio Sep 24, 2024
c77b545
Add the dependency of read-before-write to the DAG
welkin22 Sep 24, 2024
14ae50e
comments
welkin22 Sep 24, 2024
b6f55f7
txdag: add suicide checking logic;
galaio Sep 25, 2024
2bf1bb6
txdag: optimize the size of readSet
welkin22 Sep 26, 2024
6e251dc
remove remove method
welkin22 Sep 26, 2024
9c397a9
fix: incorrect index
welkin22 Sep 26, 2024
f206e6d
Remove redundant data when generating NonDependentRelFlag
welkin22 Oct 11, 2024
6a61b73
modify according to the comments
welkin22 Nov 25, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/evm/blockrunner.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func blockTestCmd(ctx *cli.Context) error {
continue
}
test := tests[name]
if err := test.Run(false, rawdb.HashScheme, tracer, func(res error, chain *core.BlockChain) {
if err := test.Run(false, rawdb.HashScheme, tracer, false, func(res error, chain *core.BlockChain) {
if ctx.Bool(DumpFlag.Name) {
if state, _ := chain.State(); state != nil {
fmt.Println(string(state.Dump(nil)))
Expand Down
2 changes: 2 additions & 0 deletions cmd/geth/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,8 @@ var (
utils.RollupComputePendingBlock,
utils.RollupHaltOnIncompatibleProtocolVersionFlag,
utils.RollupSuperchainUpgradesFlag,
utils.ParallelTxDAGFlag,
utils.ParallelTxDAGSenderPrivFlag,
configFileFlag,
utils.LogDebugFlag,
utils.LogBacktraceAtFlag,
Expand Down
24 changes: 24 additions & 0 deletions cmd/utils/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -1099,11 +1099,24 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
Category: flags.MetricsCategory,
}

ParallelTxDAGFlag = &cli.BoolFlag{
Name: "parallel.txdag",
Usage: "Enable the experimental parallel TxDAG generation (default = false)",
Category: flags.VMCategory,
}

VMOpcodeOptimizeFlag = &cli.BoolFlag{
Name: "vm.opcode.optimize",
Usage: "enable opcode optimization",
Category: flags.VMCategory,
}

ParallelTxDAGSenderPrivFlag = &cli.StringFlag{
Name: "parallel.txdagsenderpriv",
Usage: "private key of the sender who sends the TxDAG transactions",
Value: "",
Category: flags.VMCategory,
}
)

var (
Expand Down Expand Up @@ -1989,6 +2002,17 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
cfg.EnablePreimageRecording = ctx.Bool(VMEnableDebugFlag.Name)
}

if ctx.IsSet(ParallelTxDAGFlag.Name) {
cfg.EnableParallelTxDAG = ctx.Bool(ParallelTxDAGFlag.Name)
}

if ctx.IsSet(ParallelTxDAGSenderPrivFlag.Name) {
priHex := ctx.String(ParallelTxDAGSenderPrivFlag.Name)
if cfg.Miner.ParallelTxDAGSenderPriv, err = crypto.HexToECDSA(priHex); err != nil {
Fatalf("Failed to parse txdag private key of %s, err: %v", ParallelTxDAGSenderPrivFlag.Name, err)
}
}

if ctx.IsSet(VMOpcodeOptimizeFlag.Name) {
cfg.EnableOpcodeOptimizing = ctx.Bool(VMOpcodeOptimizeFlag.Name)
if cfg.EnableOpcodeOptimizing {
Expand Down
19 changes: 17 additions & 2 deletions core/blockchain.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ var (
triedbCommitExternalTimer = metrics.NewRegisteredTimer("chain/triedb/commit/external", nil)
innerExecutionTimer = metrics.NewRegisteredTimer("chain/inner/execution", nil)

txDAGGenerateTimer = metrics.NewRegisteredTimer("chain/block/txdag/gen", nil)

blockGasUsedGauge = metrics.NewRegisteredGauge("chain/block/gas/used", nil)
mgaspsGauge = metrics.NewRegisteredGauge("chain/mgas/ps", nil)

Expand Down Expand Up @@ -298,6 +300,9 @@ type BlockChain struct {
processor Processor // Block transaction processor interface
forker *ForkChoice
vmConfig vm.Config

// parallel EVM related
enableTxDAG bool
}

// NewBlockChain returns a fully initialised block chain using information
Expand Down Expand Up @@ -1987,8 +1992,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete(in validation)
accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete(in validation)
storageHashTimer.Update(statedb.StorageHashes) // Storage hashes are complete(in validation)
blockExecutionTimer.Update(ptime) // The time spent on block execution
blockValidationTimer.Update(vtime) // The time spent on block validation
txDAGGenerateTimer.Update(statedb.TxDAGGenerate)
blockExecutionTimer.Update(ptime) // The time spent on block execution
blockValidationTimer.Update(vtime) // The time spent on block validation

innerExecutionTimer.Update(DebugInnerExecutionDuration)

Expand Down Expand Up @@ -2758,3 +2764,12 @@ func createDelFn(bc *BlockChain) func(db ethdb.KeyValueWriter, hash common.Hash,
func (bc *BlockChain) HeaderChainForceSetHead(headNumber uint64) {
bc.hc.SetHead(headNumber, nil, createDelFn(bc))
}

func (bc *BlockChain) TxDAGEnabledWhenMine() bool {
return bc.enableTxDAG
}

func (bc *BlockChain) SetupTxDAGGeneration() {
log.Info("node enable TxDAG feature")
bc.enableTxDAG = true
}
2 changes: 1 addition & 1 deletion core/state/journal.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ func (ch createObjectChange) dirtied() *common.Address {
func (ch resetObjectChange) revert(s *StateDB) {
s.setStateObject(ch.prev)
if !ch.prevdestruct {
delete(s.stateObjectsDestruct, ch.prev.address)
s.removeStateObjectsDestruct(ch.prev.address)
}
if ch.prevAccount != nil {
s.accounts[ch.prev.addrHash] = ch.prevAccount
Expand Down
88 changes: 80 additions & 8 deletions core/state/state_object.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"bytes"
"fmt"
"io"
"slices"
"sync"
"time"

Expand Down Expand Up @@ -70,6 +71,11 @@ type stateObject struct {
origin *types.StateAccount // Account original data without any change applied, nil means it was not existent
data types.StateAccount // Account data with all mutations applied in the scope of block

// dirty account state
dirtyBalance *uint256.Int
dirtyNonce *uint64
dirtyCodeHash []byte

// Write caches.
trie Trie // storage trie, which becomes non-nil on first access
code Code // contract bytecode, which gets set when code is loaded
Expand All @@ -96,7 +102,7 @@ type stateObject struct {

// empty returns whether the account is considered empty.
func (s *stateObject) empty() bool {
return s.data.Nonce == 0 && s.data.Balance.IsZero() && bytes.Equal(s.data.CodeHash, types.EmptyCodeHash.Bytes())
return s.Nonce() == 0 && s.Balance().IsZero() && bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes())
}

// newObject creates a state object.
Expand All @@ -108,7 +114,7 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s
if acct == nil {
acct = types.NewEmptyStateAccount()
}
return &stateObject{
s := &stateObject{
db: db,
address: address,
addrHash: crypto.Keccak256Hash(address[:]),
Expand All @@ -119,6 +125,15 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s
dirtyStorage: make(Storage),
created: created,
}

// dirty data when create a new account
if created {
s.dirtyBalance = new(uint256.Int).Set(acct.Balance)
s.dirtyNonce = new(uint64)
*s.dirtyNonce = acct.Nonce
s.dirtyCodeHash = acct.CodeHash
}
return s
}

// EncodeRLP implements rlp.Encoder.
Expand Down Expand Up @@ -188,7 +203,7 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash {
// 1) resurrect happened, and new slot values were set -- those should
// have been handles via pendingStorage above.
// 2) we don't have new values, and can deliver empty response back
if _, destructed := s.db.stateObjectsDestruct[s.address]; destructed {
if _, destructed := s.db.getStateObjectsDestruct(s.address); destructed {
return common.Hash{}
}
// If no live objects are available, attempt to use snapshots
Expand Down Expand Up @@ -263,6 +278,19 @@ func (s *stateObject) finalise(prefetch bool) {
slotsToPrefetch = append(slotsToPrefetch, common.CopyBytes(key[:])) // Copy needed for closure
}
}

if s.dirtyNonce != nil {
s.data.Nonce = *s.dirtyNonce
s.dirtyNonce = nil
}
if s.dirtyBalance != nil {
s.data.Balance = s.dirtyBalance
s.dirtyBalance = nil
}
if s.dirtyCodeHash != nil {
s.data.CodeHash = s.dirtyCodeHash
s.dirtyCodeHash = nil
}
if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != types.EmptyRootHash {
s.db.prefetcher.prefetch(s.addrHash, s.data.Root, s.address, slotsToPrefetch)
}
Expand All @@ -271,6 +299,31 @@ func (s *stateObject) finalise(prefetch bool) {
}
}

func (s *stateObject) finaliseRWSet() {
if s.db.mvStates == nil {
return
}
ms := s.db.mvStates
for key, value := range s.dirtyStorage {
// three are some unclean dirtyStorage from previous reverted txs, it will skip finalise
// so add a new rule, if val has no change, then skip it
if value == s.GetCommittedState(key) {
continue
}
ms.RecordStorageWrite(s.address, key)
}

if s.dirtyNonce != nil && *s.dirtyNonce != s.data.Nonce {
ms.RecordAccountWrite(s.address, types.AccountNonce)
}
if s.dirtyBalance != nil && s.dirtyBalance.Cmp(s.data.Balance) != 0 {
ms.RecordAccountWrite(s.address, types.AccountBalance)
}
if s.dirtyCodeHash != nil && !slices.Equal(s.dirtyCodeHash, s.data.CodeHash) {
ms.RecordAccountWrite(s.address, types.AccountCodeHash)
}
}

// updateTrie is responsible for persisting cached storage changes into the
// object's storage trie. In case the storage trie is not yet loaded, this
// function will load the trie automatically. If any issues arise during the
Expand Down Expand Up @@ -463,13 +516,13 @@ func (s *stateObject) SubBalance(amount *uint256.Int) {
func (s *stateObject) SetBalance(amount *uint256.Int) {
s.db.journal.append(balanceChange{
account: &s.address,
prev: new(uint256.Int).Set(s.data.Balance),
prev: new(uint256.Int).Set(s.Balance()),
})
s.setBalance(amount)
}

func (s *stateObject) setBalance(amount *uint256.Int) {
s.data.Balance = amount
s.dirtyBalance = amount
}

func (s *stateObject) deepCopy(db *StateDB) *stateObject {
Expand All @@ -490,6 +543,16 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject {
obj.selfDestructed = s.selfDestructed
obj.dirtyCode = s.dirtyCode
obj.deleted = s.deleted
if s.dirtyBalance != nil {
obj.dirtyBalance = new(uint256.Int).Set(s.dirtyBalance)
}
if s.dirtyNonce != nil {
obj.dirtyNonce = new(uint64)
*obj.dirtyNonce = *s.dirtyNonce
}
if s.dirtyCodeHash != nil {
obj.dirtyCodeHash = s.dirtyCodeHash
}
return obj
}

Expand Down Expand Up @@ -547,32 +610,41 @@ func (s *stateObject) SetCode(codeHash common.Hash, code []byte) {

func (s *stateObject) setCode(codeHash common.Hash, code []byte) {
s.code = code
s.data.CodeHash = codeHash[:]
s.dirtyCodeHash = codeHash[:]
s.dirtyCode = true
compiler.GenOrLoadOptimizedCode(codeHash, s.code)
}

func (s *stateObject) SetNonce(nonce uint64) {
s.db.journal.append(nonceChange{
account: &s.address,
prev: s.data.Nonce,
prev: s.Nonce(),
})
s.setNonce(nonce)
}

func (s *stateObject) setNonce(nonce uint64) {
s.data.Nonce = nonce
s.dirtyNonce = &nonce
}

func (s *stateObject) CodeHash() []byte {
if len(s.dirtyCodeHash) > 0 {
return s.dirtyCodeHash
}
return s.data.CodeHash
}

func (s *stateObject) Balance() *uint256.Int {
if s.dirtyBalance != nil {
return s.dirtyBalance
}
return s.data.Balance
}

func (s *stateObject) Nonce() uint64 {
if s.dirtyNonce != nil {
return *s.dirtyNonce
}
return s.data.Nonce
}

Expand Down
Loading
Loading