From 1b47c48ea4ee87f92896ad784b3136025ed95a35 Mon Sep 17 00:00:00 2001 From: Max Revitt Date: Wed, 25 Sep 2024 17:40:09 +0100 Subject: [PATCH 1/2] feat(zkevm_api): move trace method to debug api closes #1228 (#1229) --- cmd/rpcdaemon/commands/daemon.go | 2 +- cmd/rpcdaemon/commands/debug_api.go | 6 +- cmd/rpcdaemon/commands/tracing_zkevm.go | 115 ++++++++++++++++++++++ cmd/rpcdaemon/commands/zkevm_api.go | 3 - cmd/rpcdaemon/commands/zkevm_counters.go | 117 ----------------------- 5 files changed, 121 insertions(+), 122 deletions(-) diff --git a/cmd/rpcdaemon/commands/daemon.go b/cmd/rpcdaemon/commands/daemon.go index 1e275aaa742..da5c5b248c0 100644 --- a/cmd/rpcdaemon/commands/daemon.go +++ b/cmd/rpcdaemon/commands/daemon.go @@ -37,7 +37,7 @@ func APIList(db kv.RoDB, borDb kv.RoDB, eth rpchelper.ApiBackend, txPool txpool. erigonImpl := NewErigonAPI(base, db, eth) txpoolImpl := NewTxPoolAPI(base, db, txPool, rawPool, rpcUrl) netImpl := NewNetAPIImpl(eth) - debugImpl := NewPrivateDebugAPI(base, db, cfg.Gascap) + debugImpl := NewPrivateDebugAPI(base, db, cfg.Gascap, ethCfg) traceImpl := NewTraceAPI(base, db, &cfg) web3Impl := NewWeb3APIImpl(eth) dbImpl := NewDBAPIImpl() /* deprecated */ diff --git a/cmd/rpcdaemon/commands/debug_api.go b/cmd/rpcdaemon/commands/debug_api.go index 5ae051690e4..6b3b2e5e28e 100644 --- a/cmd/rpcdaemon/commands/debug_api.go +++ b/cmd/rpcdaemon/commands/debug_api.go @@ -21,6 +21,7 @@ import ( "github.com/ledgerwatch/erigon/rpc" "github.com/ledgerwatch/erigon/turbo/adapter/ethapi" "github.com/ledgerwatch/erigon/turbo/transactions" + "github.com/ledgerwatch/erigon/eth/ethconfig" ) // AccountRangeMaxResults is the maximum number of results to be returned per call @@ -37,6 +38,7 @@ type PrivateDebugAPI interface { GetModifiedAccountsByHash(_ context.Context, startHash common.Hash, endHash *common.Hash) ([]common.Address, error) TraceCall(ctx context.Context, args ethapi.CallArgs, blockNrOrHash rpc.BlockNumberOrHash, config *tracers.TraceConfig_ZkEvm, stream *jsoniter.Stream) error AccountAt(ctx context.Context, blockHash common.Hash, txIndex uint64, account common.Address) (*AccountResult, error) + TraceTransactionCounters(ctx context.Context, hash common.Hash, config *tracers.TraceConfig_ZkEvm, stream *jsoniter.Stream) error } // PrivateDebugAPIImpl is implementation of the PrivateDebugAPI interface based on remote Db access @@ -44,14 +46,16 @@ type PrivateDebugAPIImpl struct { *BaseAPI db kv.RoDB GasCap uint64 + config *ethconfig.Config } // NewPrivateDebugAPI returns PrivateDebugAPIImpl instance -func NewPrivateDebugAPI(base *BaseAPI, db kv.RoDB, gascap uint64) *PrivateDebugAPIImpl { +func NewPrivateDebugAPI(base *BaseAPI, db kv.RoDB, gascap uint64, config *ethconfig.Config) *PrivateDebugAPIImpl { return &PrivateDebugAPIImpl{ BaseAPI: base, db: db, GasCap: gascap, + config: config, } } diff --git a/cmd/rpcdaemon/commands/tracing_zkevm.go b/cmd/rpcdaemon/commands/tracing_zkevm.go index 154eb371b5d..01369a89099 100644 --- a/cmd/rpcdaemon/commands/tracing_zkevm.go +++ b/cmd/rpcdaemon/commands/tracing_zkevm.go @@ -357,3 +357,118 @@ func (api *PrivateDebugAPIImpl) TraceCallMany(ctx context.Context, bundles []Bun stream.WriteArrayEnd() return nil } + +// TraceTransaction implements debug_traceTransaction. Returns Geth style transaction traces. +func (api *PrivateDebugAPIImpl) TraceTransactionCounters(ctx context.Context, hash common.Hash, config *tracers.TraceConfig_ZkEvm, stream *jsoniter.Stream) error { + tx, err := api.db.BeginRo(ctx) + if err != nil { + stream.WriteNil() + return err + } + defer tx.Rollback() + chainConfig, err := api.chainConfig(tx) + if err != nil { + stream.WriteNil() + return err + } + // Retrieve the transaction and assemble its EVM context + blockNum, ok, err := api.txnLookup(ctx, tx, hash) + if err != nil { + stream.WriteNil() + return err + } + if !ok { + stream.WriteNil() + return nil + } + + // check pruning to ensure we have history at this block level + if err = api.BaseAPI.checkPruneHistory(tx, blockNum); err != nil { + stream.WriteNil() + return err + } + + // Private API returns 0 if transaction is not found. + if blockNum == 0 && chainConfig.Bor != nil { + blockNumPtr, err := rawdb.ReadBorTxLookupEntry(tx, hash) + if err != nil { + stream.WriteNil() + return err + } + if blockNumPtr == nil { + stream.WriteNil() + return nil + } + blockNum = *blockNumPtr + } + block, err := api.blockByNumberWithSenders(tx, blockNum) + if err != nil { + stream.WriteNil() + return err + } + if block == nil { + stream.WriteNil() + return nil + } + var txnIndex uint64 + var txn types.Transaction + for i, transaction := range block.Transactions() { + if transaction.Hash() == hash { + txnIndex = uint64(i) + txn = transaction + break + } + } + if txn == nil { + borTx, _, _, _, err := rawdb.ReadBorTransaction(tx, hash) + if err != nil { + stream.WriteNil() + return err + } + + if borTx != nil { + stream.WriteNil() + return nil + } + stream.WriteNil() + return fmt.Errorf("transaction %#x not found", hash) + } + engine := api.engine() + + txEnv, err := transactions.ComputeTxEnv_ZkEvm(ctx, engine, block, chainConfig, api._blockReader, tx, int(txnIndex), api.historyV3(tx)) + if err != nil { + stream.WriteNil() + return err + } + + // counters work + hermezDb := hermez_db.NewHermezDbReader(tx) + forkId, err := hermezDb.GetForkIdByBlockNum(blockNum) + if err != nil { + stream.WriteNil() + return err + } + + smtDepth, err := getSmtDepth(hermezDb, blockNum, config) + if err != nil { + stream.WriteNil() + return err + } + + txCounters := vm.NewTransactionCounter(txn, int(smtDepth), uint16(forkId), api.config.Zk.VirtualCountersSmtReduction, false) + batchCounters := vm.NewBatchCounterCollector(int(smtDepth), uint16(forkId), api.config.Zk.VirtualCountersSmtReduction, false, nil) + + if _, err = batchCounters.AddNewTransactionCounters(txCounters); err != nil { + stream.WriteNil() + return err + } + + // set tracer to counter tracer + if config == nil { + config = &tracers.TraceConfig_ZkEvm{} + } + config.CounterCollector = txCounters.ExecutionCounters() + + // Trace the transaction and return + return transactions.TraceTx(ctx, txEnv.Msg, txEnv.BlockContext, txEnv.TxContext, txEnv.Ibs, config, chainConfig, stream, api.evmCallTimeout) +} diff --git a/cmd/rpcdaemon/commands/zkevm_api.go b/cmd/rpcdaemon/commands/zkevm_api.go index c8d2a62dd99..92d26f573c6 100644 --- a/cmd/rpcdaemon/commands/zkevm_api.go +++ b/cmd/rpcdaemon/commands/zkevm_api.go @@ -11,7 +11,6 @@ import ( libcommon "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/common/hexutility" "github.com/gateway-fm/cdk-erigon-lib/kv" - jsoniter "github.com/json-iterator/go" "github.com/holiman/uint256" "github.com/ledgerwatch/erigon/common/hexutil" @@ -20,7 +19,6 @@ import ( eritypes "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" - "github.com/ledgerwatch/erigon/eth/tracers" "github.com/ledgerwatch/erigon/rpc" "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/ledgerwatch/erigon/zk/hermez_db" @@ -61,7 +59,6 @@ type ZkEvmAPI interface { GetExitRootsByGER(ctx context.Context, globalExitRoot common.Hash) (*ZkExitRoots, error) GetL2BlockInfoTree(ctx context.Context, blockNum rpc.BlockNumberOrHash) (json.RawMessage, error) EstimateCounters(ctx context.Context, argsOrNil *zkevmRPCTransaction) (json.RawMessage, error) - TraceTransactionCounters(ctx context.Context, hash common.Hash, config *tracers.TraceConfig_ZkEvm, stream *jsoniter.Stream) error GetBatchCountersByNumber(ctx context.Context, batchNumRpc rpc.BlockNumber) (res json.RawMessage, err error) GetExitRootTable(ctx context.Context) ([]l1InfoTreeData, error) GetVersionHistory(ctx context.Context) (json.RawMessage, error) diff --git a/cmd/rpcdaemon/commands/zkevm_counters.go b/cmd/rpcdaemon/commands/zkevm_counters.go index 74d57773019..84a1df51d27 100644 --- a/cmd/rpcdaemon/commands/zkevm_counters.go +++ b/cmd/rpcdaemon/commands/zkevm_counters.go @@ -9,7 +9,6 @@ import ( "github.com/gateway-fm/cdk-erigon-lib/common/hexutility" "github.com/gateway-fm/cdk-erigon-lib/kv" "github.com/holiman/uint256" - jsoniter "github.com/json-iterator/go" "github.com/ledgerwatch/erigon/chain" "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/eth/tracers" @@ -24,7 +23,6 @@ import ( "github.com/ledgerwatch/erigon/core/vm" "github.com/ledgerwatch/erigon/core/vm/evmtypes" "github.com/ledgerwatch/erigon/turbo/rpchelper" - "github.com/ledgerwatch/erigon/turbo/transactions" "github.com/ledgerwatch/erigon/zk/hermez_db" ) @@ -272,121 +270,6 @@ func populateCounters(collected *vm.Counters, execResult *core.ExecutionResult, return resJson, nil } -// TraceTransaction implements debug_traceTransaction. Returns Geth style transaction traces. -func (api *ZkEvmAPIImpl) TraceTransactionCounters(ctx context.Context, hash common.Hash, config *tracers.TraceConfig_ZkEvm, stream *jsoniter.Stream) error { - tx, err := api.db.BeginRo(ctx) - if err != nil { - stream.WriteNil() - return err - } - defer tx.Rollback() - chainConfig, err := api.ethApi.chainConfig(tx) - if err != nil { - stream.WriteNil() - return err - } - // Retrieve the transaction and assemble its EVM context - blockNum, ok, err := api.ethApi.txnLookup(ctx, tx, hash) - if err != nil { - stream.WriteNil() - return err - } - if !ok { - stream.WriteNil() - return nil - } - - // check pruning to ensure we have history at this block level - if err = api.ethApi.BaseAPI.checkPruneHistory(tx, blockNum); err != nil { - stream.WriteNil() - return err - } - - // Private API returns 0 if transaction is not found. - if blockNum == 0 && chainConfig.Bor != nil { - blockNumPtr, err := rawdb.ReadBorTxLookupEntry(tx, hash) - if err != nil { - stream.WriteNil() - return err - } - if blockNumPtr == nil { - stream.WriteNil() - return nil - } - blockNum = *blockNumPtr - } - block, err := api.ethApi.blockByNumberWithSenders(tx, blockNum) - if err != nil { - stream.WriteNil() - return err - } - if block == nil { - stream.WriteNil() - return nil - } - var txnIndex uint64 - var txn types.Transaction - for i, transaction := range block.Transactions() { - if transaction.Hash() == hash { - txnIndex = uint64(i) - txn = transaction - break - } - } - if txn == nil { - borTx, _, _, _, err := rawdb.ReadBorTransaction(tx, hash) - if err != nil { - stream.WriteNil() - return err - } - - if borTx != nil { - stream.WriteNil() - return nil - } - stream.WriteNil() - return fmt.Errorf("transaction %#x not found", hash) - } - engine := api.ethApi.engine() - - txEnv, err := transactions.ComputeTxEnv_ZkEvm(ctx, engine, block, chainConfig, api.ethApi._blockReader, tx, int(txnIndex), api.ethApi.historyV3(tx)) - if err != nil { - stream.WriteNil() - return err - } - - // counters work - hermezDb := hermez_db.NewHermezDbReader(tx) - forkId, err := hermezDb.GetForkIdByBlockNum(blockNum) - if err != nil { - stream.WriteNil() - return err - } - - smtDepth, err := getSmtDepth(hermezDb, blockNum, config) - if err != nil { - stream.WriteNil() - return err - } - - txCounters := vm.NewTransactionCounter(txn, int(smtDepth), uint16(forkId), api.config.Zk.VirtualCountersSmtReduction, false) - batchCounters := vm.NewBatchCounterCollector(int(smtDepth), uint16(forkId), api.config.Zk.VirtualCountersSmtReduction, false, nil) - - if _, err = batchCounters.AddNewTransactionCounters(txCounters); err != nil { - stream.WriteNil() - return err - } - - // set tracer to counter tracer - if config == nil { - config = &tracers.TraceConfig_ZkEvm{} - } - config.CounterCollector = txCounters.ExecutionCounters() - - // Trace the transaction and return - return transactions.TraceTx(ctx, txEnv.Msg, txEnv.BlockContext, txEnv.TxContext, txEnv.Ibs, config, chainConfig, stream, api.ethApi.evmCallTimeout) -} - func getSmtDepth(hermezDb *hermez_db.HermezDbReader, blockNum uint64, config *tracers.TraceConfig_ZkEvm) (int, error) { var smtDepth int if config != nil && config.SmtDepth != nil { From da9bb343d72d760fdbce819c95aca3af12350e7c Mon Sep 17 00:00:00 2001 From: Scott Fairclough <70711990+hexoscott@users.noreply.github.com> Date: Thu, 26 Sep 2024 13:18:08 +0100 Subject: [PATCH 2/2] fix off by one issue in overwriting transactions (#1234) # Conflicts: # core/rawdb/accessors_chain_zkevm.go --- core/rawdb/accessors_chain_zkevm.go | 34 ++++++++- core/rawdb/accessors_chain_zkevm_test.go | 95 ++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 core/rawdb/accessors_chain_zkevm_test.go diff --git a/core/rawdb/accessors_chain_zkevm.go b/core/rawdb/accessors_chain_zkevm.go index 905a9e71696..0a05057994f 100644 --- a/core/rawdb/accessors_chain_zkevm.go +++ b/core/rawdb/accessors_chain_zkevm.go @@ -1,6 +1,7 @@ package rawdb import ( + "bytes" "encoding/binary" "fmt" @@ -9,6 +10,7 @@ import ( "github.com/gateway-fm/cdk-erigon-lib/common/hexutility" "github.com/gateway-fm/cdk-erigon-lib/kv" "github.com/gateway-fm/cdk-erigon-lib/kv/kvcfg" + "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/dbutils" "github.com/ledgerwatch/erigon/core/types" ethTypes "github.com/ledgerwatch/erigon/core/types" @@ -83,9 +85,9 @@ func WriteBodyAndTransactions(db kv.RwTx, hash libcommon.Hash, number uint64, tx } transactionV3, _ := kvcfg.TransactionsV3.Enabled(db.(kv.Tx)) if transactionV3 { - err = WriteTransactions(db, txs, data.BaseTxId+1, &hash) + err = OverwriteTransactions(db, txs, data.BaseTxId+1, &hash) } else { - err = WriteTransactions(db, txs, data.BaseTxId+1, nil) + err = OverwriteTransactions(db, txs, data.BaseTxId+1, nil) } if err != nil { return fmt.Errorf("failed to WriteTransactions: %w", err) @@ -93,6 +95,34 @@ func WriteBodyAndTransactions(db kv.RwTx, hash libcommon.Hash, number uint64, tx return nil } +func OverwriteTransactions(db kv.RwTx, txs []types.Transaction, baseTxId uint64, blockHash *libcommon.Hash) error { + txId := baseTxId + buf := bytes.NewBuffer(nil) + for _, tx := range txs { + txIdKey := make([]byte, 8) + binary.BigEndian.PutUint64(txIdKey, txId) + txId++ + + buf.Reset() + if err := rlp.Encode(buf, tx); err != nil { + return fmt.Errorf("broken tx rlp: %w", err) + } + + // If next Append returns KeyExists error - it means you need to open transaction in App code before calling this func. Batch is also fine. + if blockHash != nil { + key := append(txIdKey, blockHash.Bytes()...) + if err := db.Put(kv.EthTxV3, key, common.CopyBytes(buf.Bytes())); err != nil { + return err + } + } else { + if err := db.Put(kv.EthTx, txIdKey, common.CopyBytes(buf.Bytes())); err != nil { + return err + } + } + } + return nil +} + func GetBodyTransactions(tx kv.RwTx, fromBlockNum, toBlockNum uint64) (*[]types.Transaction, error) { var transactions []types.Transaction if err := tx.ForEach(kv.BlockBody, hexutility.EncodeTs(fromBlockNum), func(k, v []byte) error { diff --git a/core/rawdb/accessors_chain_zkevm_test.go b/core/rawdb/accessors_chain_zkevm_test.go new file mode 100644 index 00000000000..7eefc1b8c56 --- /dev/null +++ b/core/rawdb/accessors_chain_zkevm_test.go @@ -0,0 +1,95 @@ +package rawdb + +import ( + "testing" + "github.com/gateway-fm/cdk-erigon-lib/kv/memdb" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/sha3" + libcommon "github.com/gateway-fm/cdk-erigon-lib/common" + "github.com/ledgerwatch/erigon/crypto" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/common/u256" + "github.com/ledgerwatch/erigon/params" + "github.com/ledgerwatch/erigon/rlp" + "github.com/ledgerwatch/erigon/common/dbutils" +) + +func TestBodyStorageZkevm(t *testing.T) { + _, tx := memdb.NewTestTx(t) + require := require.New(t) + + var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + mustSign := func(tx types.Transaction, s types.Signer) types.Transaction { + r, err := types.SignTx(tx, s, testKey) + require.NoError(err) + return r + } + + // prepare db so it works with our test + signer1 := types.MakeSigner(params.HermezMainnetChainConfig, 1) + body := &types.Body{ + Transactions: []types.Transaction{ + mustSign(types.NewTransaction(1, testAddr, u256.Num1, 1, u256.Num1, nil), *signer1), + mustSign(types.NewTransaction(2, testAddr, u256.Num1, 2, u256.Num1, nil), *signer1), + }, + Uncles: []*types.Header{{Extra: []byte("test header")}}, + } + + // Create a test body to move around the database and make sure it's really new + hasher := sha3.NewLegacyKeccak256() + _ = rlp.Encode(hasher, body) + hash := libcommon.BytesToHash(hasher.Sum(nil)) + + if entry := ReadCanonicalBodyWithTransactions(tx, hash, 0); entry != nil { + t.Fatalf("Non existent body returned: %v", entry) + } + require.NoError(WriteBody(tx, hash, 0, body)) + if entry := ReadCanonicalBodyWithTransactions(tx, hash, 0); entry == nil { + t.Fatalf("Stored body not found") + } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(types.Transactions(body.Transactions)) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) { + t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, body) + } + if entry := ReadBodyRLP(tx, hash, 0); entry == nil { + t.Fatalf("Stored body RLP not found") + } else { + hasher := sha3.NewLegacyKeccak256() + hasher.Write(entry) + + if calc := libcommon.BytesToHash(hasher.Sum(nil)); calc != hash { + t.Fatalf("Retrieved RLP body mismatch: have %v, want %v", entry, body) + } + } + + // zkevm check with overwriting transactions + bodyForStorage, err := ReadBodyForStorageByKey(tx, dbutils.BlockBodyKey(0, hash)) + if err != nil { + t.Fatalf("ReadBodyForStorageByKey failed: %s", err) + } + // overwrite the transactions using the new code from zkevm + require.NoError(WriteBodyAndTransactions(tx, hash, 0, body.Transactions, bodyForStorage)) + + // now re-run the checks from above after reading the body again + if entry := ReadCanonicalBodyWithTransactions(tx, hash, 0); entry == nil { + t.Fatalf("Stored body not found") + } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(types.Transactions(body.Transactions)) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) { + t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, body) + } + if entry := ReadBodyRLP(tx, hash, 0); entry == nil { + t.Fatalf("Stored body RLP not found") + } else { + hasher := sha3.NewLegacyKeccak256() + hasher.Write(entry) + + if calc := libcommon.BytesToHash(hasher.Sum(nil)); calc != hash { + t.Fatalf("Retrieved RLP body mismatch: have %v, want %v", entry, body) + } + } + + // Delete the body and verify the execution + deleteBody(tx, hash, 0) + if entry := ReadCanonicalBodyWithTransactions(tx, hash, 0); entry != nil { + t.Fatalf("Deleted body returned: %v", entry) + } +}