From d000cebf4cc5ca0792e599cffc8d5ca39e4e5f94 Mon Sep 17 00:00:00 2001 From: Valentin Staykov <79150443+V-Staykov@users.noreply.github.com> Date: Mon, 5 Aug 2024 13:33:41 +0300 Subject: [PATCH] raised size of memdb (#915) * raised size of memdb * added datasize flag * update erigon-lib * addded libs --- cmd/rpcdaemon/commands/zkevm_api.go | 1 + cmd/utils/customflags_zkevm.go | 86 +++++++++++++++++++++++++++++ cmd/utils/flags.go | 6 ++ eth/backend.go | 1 + eth/ethconfig/config_zkevm.go | 2 + go.mod | 4 +- go.sum | 18 ++++++ smt/pkg/smt/smt_batch.go | 49 ++++++++++------ turbo/cli/default_flags.go | 1 + turbo/cli/flags_zkevm.go | 3 + zk/witness/witness.go | 30 +++++----- 11 files changed, 168 insertions(+), 33 deletions(-) create mode 100644 cmd/utils/customflags_zkevm.go diff --git a/cmd/rpcdaemon/commands/zkevm_api.go b/cmd/rpcdaemon/commands/zkevm_api.go index 0e36d4fba58..16ee3284654 100644 --- a/cmd/rpcdaemon/commands/zkevm_api.go +++ b/cmd/rpcdaemon/commands/zkevm_api.go @@ -849,6 +849,7 @@ func (api *ZkEvmAPIImpl) buildGenerator(tx kv.Tx, witnessMode WitnessMode) (*wit api.ethApi._agg, api.ethApi._blockReader, chainConfig, + api.config.Zk, api.ethApi._engine, ) diff --git a/cmd/utils/customflags_zkevm.go b/cmd/utils/customflags_zkevm.go new file mode 100644 index 00000000000..069a60a378e --- /dev/null +++ b/cmd/utils/customflags_zkevm.go @@ -0,0 +1,86 @@ +package utils + +import ( + "flag" + "fmt" + + "github.com/c2h5oh/datasize" + "github.com/urfave/cli/v2" +) + +// Custom cli.Flag type which expand the received string to an absolute path. +// e.g. ~/.ethereum -> /home/username/.ethereum +type DatasizeFlag struct { + Name string + + Category string + DefaultText string + Usage string + + Required bool + Hidden bool + HasBeenSet bool + + Value datasizeFlagValue + + Aliases []string +} + +func (f *DatasizeFlag) Names() []string { return append([]string{f.Name}, f.Aliases...) } +func (f *DatasizeFlag) IsSet() bool { return f.HasBeenSet } +func (f *DatasizeFlag) String() string { return cli.FlagStringer(f) } + +// called by cli library, grabs variable from environment (if in env) +// and adds variable to flag set for parsing. +func (f *DatasizeFlag) Apply(set *flag.FlagSet) error { + eachName(f, func(name string) { + set.Var(&f.Value, f.Name, f.Usage) + }) + return nil +} + +func (f *DatasizeFlag) IsRequired() bool { return f.Required } + +func (f *DatasizeFlag) IsVisible() bool { return !f.Hidden } + +func (f *DatasizeFlag) GetCategory() string { return f.Category } + +func (f *DatasizeFlag) TakesValue() bool { return true } +func (f *DatasizeFlag) GetUsage() string { return f.Usage } +func (f *DatasizeFlag) GetValue() string { return f.Value.String() } +func (f *DatasizeFlag) GetEnvVars() []string { return nil } // env not supported + +func (f *DatasizeFlag) GetDefaultText() string { + if f.DefaultText != "" { + return f.DefaultText + } + return f.GetValue() +} + +type datasizeFlagValue datasize.ByteSize + +func (b *datasizeFlagValue) String() string { + if b == nil { + return "" + } + a := datasize.ByteSize(*b) + return a.String() +} + +func (b *datasizeFlagValue) Set(s string) error { + val, err := datasize.ParseString(s) + if err != nil { + return fmt.Errorf("parse datasize: %v", err) + } + *b = datasizeFlagValue(val) + return nil +} + +// DatasizeFlagValue returns the value of a DatasizeFlag from the flag set. +func DatasizeFlagValue(ctx *cli.Context, name string) *datasize.ByteSize { + val := ctx.Generic(name) + if val == nil { + return nil + } + return (*datasize.ByteSize)(val.(*datasizeFlagValue)) +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 2134e156cd5..144a8c8de10 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -501,6 +501,12 @@ var ( Usage: "The timeout for the executor request", Value: 500 * time.Millisecond, } + + WitnessMemdbSize = DatasizeFlag{ + Name: "zkevm.witness-memdb-size", + Usage: "A size of the memdb used on witness generation in format \"2GB\". Might fail generation for older batches if not enough for the unwind.", + Value: datasizeFlagValue(2 * datasize.GB), + } ExecutorMaxConcurrentRequests = cli.IntFlag{ Name: "zkevm.executor-max-concurrent-requests", Usage: "The maximum number of concurrent requests to the executor", diff --git a/eth/backend.go b/eth/backend.go index 19f9cce99b5..1a79ad0900b 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -829,6 +829,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { backend.agg, backend.blockReader, backend.chainConfig, + backend.config.Zk, backend.engine, ) diff --git a/eth/ethconfig/config_zkevm.go b/eth/ethconfig/config_zkevm.go index ca9b56df1c2..5efc0421224 100644 --- a/eth/ethconfig/config_zkevm.go +++ b/eth/ethconfig/config_zkevm.go @@ -3,6 +3,7 @@ package ethconfig import ( "time" + "github.com/c2h5oh/datasize" "github.com/gateway-fm/cdk-erigon-lib/common" ) @@ -36,6 +37,7 @@ type Zk struct { ExecutorStrictMode bool ExecutorRequestTimeout time.Duration DatastreamNewBlockTimeout time.Duration + WitnessMemdbSize datasize.ByteSize ExecutorMaxConcurrentRequests int Limbo bool AllowFreeTransactions bool diff --git a/go.mod b/go.mod index 814bbb63ade..77a2444a55c 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/emicklei/dot v1.0.0 github.com/emirpasic/gods v1.18.1 github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c - github.com/gateway-fm/cdk-erigon-lib v0.0.0-20240415152010-66aa405ca33f + github.com/gateway-fm/cdk-erigon-lib v0.0.0-20240805074757-5e3196248fde github.com/gateway-fm/vectorized-poseidon-gold v1.0.0 github.com/gballet/go-verkle v0.0.0-20221121182333-31427a1f2d35 github.com/go-stack/stack v1.8.1 @@ -182,6 +182,7 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/ledgerwatch/interfaces v0.0.0-20230412092010-e1c4a1a4279e // indirect github.com/lib/pq v1.10.7 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect @@ -197,6 +198,7 @@ require ( github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect + github.com/matryer/moq v0.3.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect diff --git a/go.sum b/go.sum index e74fb82299d..f690c7f6268 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU= +filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3-RC4 h1:+4K+xSzv0ImbK30B/T9FauNTrTFUmWcNKYhIgwsE4C4= github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3-RC4/go.mod h1:0QkAXcFa92mFJrCbN3UPUJGJYes851yEgYHLONnaosE= @@ -45,6 +46,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/anacrolix/chansync v0.3.0 h1:lRu9tbeuw3wl+PhMu/r+JJCRu5ArFXIluOgdF0ao6/U= @@ -56,6 +58,7 @@ github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54g github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= github.com/anacrolix/envpprof v1.2.1 h1:25TJe6t/i0AfzzldiGFKCpD+s+dk8lONBcacJZB2rdE= github.com/anacrolix/envpprof v1.2.1/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= +github.com/anacrolix/fuse v0.2.0/go.mod h1:Kfu02xBwnySDpH3N23BmrP3MDfwAQGRLUCj6XyeOvBQ= github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60 h1:k4/h2B1gGF+PJGyGHxs8nmHHt1pzWXZWBj6jn4OBlRc= github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= github.com/anacrolix/go-libutp v1.2.0 h1:sjxoB+/ARiKUR7IK/6wLWyADIBqGmu1fm0xo+8Yy7u0= @@ -86,6 +89,7 @@ github.com/anacrolix/mmsg v1.0.0 h1:btC7YLjOn29aTUAExJiVUhQOuf/8rhm+/nWCMAnL3Hg= github.com/anacrolix/mmsg v1.0.0/go.mod h1:x8kRaJY/dCrY9Al0PEcj1mb/uFHwP6GCJ9fLl4thEPc= github.com/anacrolix/multiless v0.3.0 h1:5Bu0DZncjE4e06b9r1Ap2tUY4Au0NToBP5RpuEngSis= github.com/anacrolix/multiless v0.3.0/go.mod h1:TrCLEZfIDbMVfLoQt5tOoiBS/uq4y8+ojuEVVvTNPX4= +github.com/anacrolix/publicip v0.2.0/go.mod h1:67G1lVkLo8UjdEcJkwScWVTvlJ35OCDsRJoWXl/wi4g= github.com/anacrolix/stm v0.2.0/go.mod h1:zoVQRvSiGjGoTmbM0vSLIiaKjWtNPeTvXUSdJQA4hsg= github.com/anacrolix/stm v0.4.0 h1:tOGvuFwaBjeu1u9X1eIh9TX8OEedEiEQ1se1FjhFnXY= github.com/anacrolix/stm v0.4.0/go.mod h1:GCkwqWoAsP7RfLW+jw+Z0ovrt2OO7wRzcTtFYMYY5t8= @@ -96,6 +100,7 @@ github.com/anacrolix/sync v0.4.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DC github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= +github.com/anacrolix/tagflag v1.3.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= github.com/anacrolix/torrent v1.48.1-0.20230219022425-e8971ea0f1bf h1:gQCApNMI+lbXYLRiiiC5S2mU9k2BZT9FNnRr//eUzXc= github.com/anacrolix/torrent v1.48.1-0.20230219022425-e8971ea0f1bf/go.mod h1:5OY82KVPu5Fq+P0HefdTQKRt0gfBXeHeRUE04VaSoQo= github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96 h1:QAVZ3pN/J4/UziniAhJR2OZ9Ox5kOY2053tBbbqUPYA= @@ -181,6 +186,7 @@ github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS3 github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g= github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/dgravesa/go-parallel v0.6.0 h1:3KU5uzHn92Upq9jPVQ+ILLechgUKigCo72TScC7U3qE= @@ -213,6 +219,7 @@ github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8E github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys= github.com/emicklei/dot v1.0.0 h1:yyObALINBOuI1GdCRwVea2IPtGtVgh0NQgJDrE03Tqc= github.com/emicklei/dot v1.0.0/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -240,6 +247,8 @@ github.com/garslo/gogen v0.0.0-20170307003452-d6ebae628c7c h1:uYNKzPntb8c6DKvP9E github.com/garslo/gogen v0.0.0-20170307003452-d6ebae628c7c/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/gateway-fm/cdk-erigon-lib v0.0.0-20240415152010-66aa405ca33f h1:/o6bj7zF7c72CttEswr8gG0iFC749eVprCVc2pKIB9w= github.com/gateway-fm/cdk-erigon-lib v0.0.0-20240415152010-66aa405ca33f/go.mod h1:Ky//z32wQOTY/drV858kr8dECBD9OBV9Xzy+aG6TxSk= +github.com/gateway-fm/cdk-erigon-lib v0.0.0-20240805074757-5e3196248fde h1:0myaeF/FBykEQx49FLX/ERd9EVnPh5yV1GwRu+9uLAk= +github.com/gateway-fm/cdk-erigon-lib v0.0.0-20240805074757-5e3196248fde/go.mod h1:Ky//z32wQOTY/drV858kr8dECBD9OBV9Xzy+aG6TxSk= github.com/gateway-fm/vectorized-poseidon-gold v1.0.0 h1:Du0ZW+fkZhgRNGx/gAkHnMj3/Rl8uJkAEe+ZDPX3PDw= github.com/gateway-fm/vectorized-poseidon-gold v1.0.0/go.mod h1:VLGQpyjrOg8+FugH/+d8tfYd/c3z4Xqa+zbUBITygaw= github.com/gballet/go-verkle v0.0.0-20221121182333-31427a1f2d35 h1:I8QswD9gf3VEpr7bpepKKOm7ChxFITIG+oc1I5/S0no= @@ -264,6 +273,7 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -472,6 +482,8 @@ github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+ github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -520,6 +532,8 @@ github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336 h1:Yxmt4Wyd0RCLr7UJJAl0ApCP/f5qkWfvHfgPbnI8ghM= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/interfaces v0.0.0-20230412092010-e1c4a1a4279e h1:mT6GE/XsuUVQGTcZjrq0KoINds2fKa8VsHhGbe2PF54= +github.com/ledgerwatch/interfaces v0.0.0-20230412092010-e1c4a1a4279e/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= github.com/ledgerwatch/log/v3 v3.7.0/go.mod h1:J2Jl6zV/58LeA6LTaVVnCGyf1/cYYSEOOLHY4ZN8S2A= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= @@ -575,6 +589,8 @@ github.com/maticnetwork/crand v1.0.2 h1:Af0tAivC8zrxXDpGWNWVT/0s1fOz8w0eRbahZgUR github.com/maticnetwork/crand v1.0.2/go.mod h1:/NRNL3bj2eYdqpWmoIP5puxndTpi0XRxpj5ZKxfHjyg= github.com/maticnetwork/polyproto v0.0.2 h1:cPxuxbIDItdwGnucc3lZB58U8Zfe1mH73PWTGd15554= github.com/maticnetwork/polyproto v0.0.2/go.mod h1:e1mU2EXSwEpn5jM7GfNwu3AupsV6WAGoPFFfswXOF0o= +github.com/matryer/moq v0.3.1 h1:kLDiBJoGcusWS2BixGyTkF224aSCD8nLY24tj/NcTCs= +github.com/matryer/moq v0.3.1/go.mod h1:RJ75ZZZD71hejp39j4crZLsEDszGk6iH4v4YsWFKH4s= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -647,6 +663,7 @@ github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 h1:dOYG7LS/WK00RWZc8XGgcUTlTxpp3mKhdR2Q9z9HbXM= @@ -1269,6 +1286,7 @@ modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY= modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= diff --git a/smt/pkg/smt/smt_batch.go b/smt/pkg/smt/smt_batch.go index 5d5209e7a27..eb924c03a68 100644 --- a/smt/pkg/smt/smt_batch.go +++ b/smt/pkg/smt/smt_batch.go @@ -26,25 +26,21 @@ func (s *SMT) InsertBatch(ctx context.Context, logPrefix string, nodeKeys []*uti progressChan, stopProgressPrinter := zk.ProgressPrinterWithoutValues(fmt.Sprintf("[%s] SMT incremental progress", logPrefix), uint64(size)+preprocessStage+finalizeStage) defer stopProgressPrinter() - err = validateDataLengths(nodeKeys, nodeValues, &nodeValuesHashes) - if err != nil { + if err = validateDataLengths(nodeKeys, nodeValues, &nodeValuesHashes); err != nil { return nil, err } - err = removeDuplicateEntriesByKeys(&size, &nodeKeys, &nodeValues, &nodeValuesHashes) - if err != nil { + if err = removeDuplicateEntriesByKeys(&size, &nodeKeys, &nodeValues, &nodeValuesHashes); err != nil { return nil, err } - err = calculateNodeValueHashesIfMissing(s, nodeValues, &nodeValuesHashes) - if err != nil { + if err = calculateNodeValueHashesIfMissing(s, nodeValues, &nodeValuesHashes); err != nil { return nil, err } progressChan <- uint64(preprocessStage) - err = calculateRootNodeHashIfNil(s, &rootNodeHash) - if err != nil { + if err = calculateRootNodeHashIfNil(s, &rootNodeHash); err != nil { return nil, err } @@ -172,7 +168,9 @@ func (s *SMT) InsertBatch(ctx context.Context, logPrefix string, nodeKeys []*uti if smtBatchNodeRoot == nil { rootNodeHash = &utils.NodeKey{0, 0, 0, 0} } else { - calculateAndSaveHashesDfs(s, smtBatchNodeRoot, make([]int, 256), 0) + if err := calculateAndSaveHashesDfs(s, smtBatchNodeRoot, make([]int, 256), 0); err != nil { + return nil, fmt.Errorf("calculating and saving hashes dfs: %w", err) + } rootNodeHash = (*utils.NodeKey)(smtBatchNodeRoot.hash) } if err := s.setLastRoot(*rootNodeHash); err != nil { @@ -386,13 +384,16 @@ func calculateAndSaveHashesDfs(s *SMT, smtBatchNode *smtBatchNode, path []int, l if smtBatchNode.isLeaf() { hashObj, err := s.hashcalcAndSave(utils.ConcatArrays4(*smtBatchNode.nodeLeftHashOrRemainingKey, *smtBatchNode.nodeRightHashOrValueHash), utils.LeafCapacity) if err != nil { - return err + return fmt.Errorf("hashing leaf: %w", err) } + smtBatchNode.hash = &hashObj + nodeKey := utils.JoinKey(path[:level], *smtBatchNode.nodeLeftHashOrRemainingKey) - s.Db.InsertHashKey(hashObj, *nodeKey) + if err := s.Db.InsertHashKey(hashObj, *nodeKey); err != nil { + return fmt.Errorf("inserting hash key: %w", err) + } - smtBatchNode.hash = &hashObj return nil } @@ -400,18 +401,30 @@ func calculateAndSaveHashesDfs(s *SMT, smtBatchNode *smtBatchNode, path []int, l if smtBatchNode.leftNode != nil { path[level] = 0 - calculateAndSaveHashesDfs(s, smtBatchNode.leftNode, path, level+1) - totalHash.SetHalfValue(*smtBatchNode.leftNode.hash, 0) + if err := calculateAndSaveHashesDfs(s, smtBatchNode.leftNode, path, level+1); err != nil { + return err + } + if err := totalHash.SetHalfValue(*smtBatchNode.leftNode.hash, 0); err != nil { + return err + } } else { - totalHash.SetHalfValue(*smtBatchNode.nodeLeftHashOrRemainingKey, 0) + if err := totalHash.SetHalfValue(*smtBatchNode.nodeLeftHashOrRemainingKey, 0); err != nil { + return err + } } if smtBatchNode.rightNode != nil { path[level] = 1 - calculateAndSaveHashesDfs(s, smtBatchNode.rightNode, path, level+1) - totalHash.SetHalfValue(*smtBatchNode.rightNode.hash, 1) + if err := calculateAndSaveHashesDfs(s, smtBatchNode.rightNode, path, level+1); err != nil { + return err + } + if err := totalHash.SetHalfValue(*smtBatchNode.rightNode.hash, 1); err != nil { + return err + } } else { - totalHash.SetHalfValue(*smtBatchNode.nodeRightHashOrValueHash, 1) + if err := totalHash.SetHalfValue(*smtBatchNode.nodeRightHashOrValueHash, 1); err != nil { + return err + } } hashObj, err := s.hashcalcAndSave(totalHash.ToUintArray(), utils.BranchCapacity) diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go index dbaa2fab9f2..d9b77ca91d0 100644 --- a/turbo/cli/default_flags.go +++ b/turbo/cli/default_flags.go @@ -197,6 +197,7 @@ var DefaultFlags = []cli.Flag{ &utils.ExecutorStrictMode, &utils.ExecutorRequestTimeout, &utils.DatastreamNewBlockTimeout, + &utils.WitnessMemdbSize, &utils.ExecutorMaxConcurrentRequests, &utils.Limbo, &utils.AllowFreeTransactions, diff --git a/turbo/cli/flags_zkevm.go b/turbo/cli/flags_zkevm.go index b03d59e15af..caa6a8d88b9 100644 --- a/turbo/cli/flags_zkevm.go +++ b/turbo/cli/flags_zkevm.go @@ -99,6 +99,8 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) { panic("Effective gas price for contract deployment must be in interval [0; 1]") } + witnessMemSize := utils.DatasizeFlagValue(ctx, utils.WitnessMemdbSize.Name) + cfg.Zk = ðconfig.Zk{ L2ChainId: ctx.Uint64(utils.L2ChainIdFlag.Name), L2RpcUrl: ctx.String(utils.L2RpcUrlFlag.Name), @@ -132,6 +134,7 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) { ExecutorStrictMode: ctx.Bool(utils.ExecutorStrictMode.Name), ExecutorRequestTimeout: ctx.Duration(utils.ExecutorRequestTimeout.Name), DatastreamNewBlockTimeout: ctx.Duration(utils.DatastreamNewBlockTimeout.Name), + WitnessMemdbSize: *witnessMemSize, ExecutorMaxConcurrentRequests: ctx.Int(utils.ExecutorMaxConcurrentRequests.Name), Limbo: ctx.Bool(utils.Limbo.Name), AllowFreeTransactions: ctx.Bool(utils.AllowFreeTransactions.Name), diff --git a/zk/witness/witness.go b/zk/witness/witness.go index 6afc14f9a2a..20cd7652b27 100644 --- a/zk/witness/witness.go +++ b/zk/witness/witness.go @@ -21,6 +21,7 @@ import ( "github.com/ledgerwatch/erigon/core/systemcontracts" eritypes "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/stagedsync" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" db2 "github.com/ledgerwatch/erigon/smt/pkg/db" @@ -48,6 +49,7 @@ type Generator struct { agg *libstate.AggregatorV3 blockReader services.FullBlockReader chainCfg *chain.Config + zkConfig *ethconfig.Zk engine consensus.EngineReader } @@ -57,6 +59,7 @@ func NewGenerator( agg *libstate.AggregatorV3, blockReader services.FullBlockReader, chainCfg *chain.Config, + zkConfig *ethconfig.Zk, engine consensus.EngineReader, ) *Generator { return &Generator{ @@ -65,11 +68,12 @@ func NewGenerator( agg: agg, blockReader: blockReader, chainCfg: chainCfg, + zkConfig: zkConfig, engine: engine, } } -func (g *Generator) GetWitnessByBatch(tx kv.Tx, ctx context.Context, batchNum uint64, debug, witnessFull bool) ([]byte, error) { +func (g *Generator) GetWitnessByBatch(tx kv.Tx, ctx context.Context, batchNum uint64, debug, witnessFull bool) (witness []byte, err error) { t := zkUtils.StartTimer("witness", "getwitnessbybatch") defer t.LogTimer() @@ -78,10 +82,12 @@ func (g *Generator) GetWitnessByBatch(tx kv.Tx, ctx context.Context, batchNum ui if err != nil { return nil, err } - var witness []byte if badBatch { // we need the header of the block prior to this batch to build up the blocks previousHeight, err := reader.GetHighestBlockInBatch(batchNum - 1) + if err != nil { + return nil, err + } previousHeader := rawdb.ReadHeaderByNumber(tx, previousHeight) if previousHeader == nil { return nil, fmt.Errorf("failed to get header for block %d", previousHeight) @@ -120,8 +126,7 @@ func (g *Generator) GetWitnessByBatch(tx kv.Tx, ctx context.Context, batchNum ui blocks[i] = block } - witness, err = g.generateWitness(tx, ctx, blocks, debug, witnessFull) - + return g.generateWitness(tx, ctx, blocks, debug, witnessFull) } else { blockNumbers, err := reader.GetL2BlockNosByBatch(batchNum) if err != nil { @@ -140,10 +145,8 @@ func (g *Generator) GetWitnessByBatch(tx kv.Tx, ctx context.Context, batchNum ui blocks[idx] = block idx++ } - witness, err = g.generateWitness(tx, ctx, blocks, debug, witnessFull) + return g.generateWitness(tx, ctx, blocks, debug, witnessFull) } - - return witness, nil } func (g *Generator) GetWitnessByBlockRange(tx kv.Tx, ctx context.Context, startBlock, endBlock uint64, debug, witnessFull bool) ([]byte, error) { @@ -187,7 +190,7 @@ func (g *Generator) generateWitness(tx kv.Tx, ctx context.Context, blocks []*eri return nil, fmt.Errorf("block number is in the future latest=%d requested=%d", latestBlock, endBlock) } - batch := memdb.NewMemoryBatch(tx, g.dirs.Tmp) + batch := memdb.NewMemoryBatchWithSize(tx, g.dirs.Tmp, g.zkConfig.WitnessMemdbSize) defer batch.Rollback() if err = populateDbTables(batch); err != nil { return nil, err @@ -209,14 +212,13 @@ func (g *Generator) generateWitness(tx kv.Tx, ctx context.Context, blocks []*eri hashStageCfg := stagedsync.StageHashStateCfg(nil, g.dirs, g.historyV3, g.agg) hashStageCfg.SetQuiet(true) if err := stagedsync.UnwindHashStateStage(unwindState, stageState, batch, hashStageCfg, ctx); err != nil { - return nil, err + return nil, fmt.Errorf("unwind hash state: %w", err) } interHashStageCfg := zkStages.StageZkInterHashesCfg(nil, true, true, false, g.dirs.Tmp, g.blockReader, nil, g.historyV3, g.agg, nil) - err = zkStages.UnwindZkIntermediateHashesStage(unwindState, stageState, batch, interHashStageCfg, ctx) - if err != nil { - return nil, err + if err = zkStages.UnwindZkIntermediateHashesStage(unwindState, stageState, batch, interHashStageCfg, ctx); err != nil { + return nil, fmt.Errorf("unwind intermediate hashes: %w", err) } tx = batch @@ -334,7 +336,7 @@ func (g *Generator) generateWitness(tx kv.Tx, ctx context.Context, blocks []*eri witness, err := smt.BuildWitness(smtTrie, rl, ctx) if err != nil { - return nil, err + return nil, fmt.Errorf("build witness: %v", err) } return getWitnessBytes(witness, debug) @@ -349,7 +351,7 @@ func getWitnessBytes(witness *trie.Witness, debug bool) ([]byte, error) { return buf.Bytes(), nil } -func populateDbTables(batch *memdb.MemoryMutation) error { +func populateDbTables(batch kv.RwTx) error { tables := []string{ db2.TableSmt, db2.TableAccountValues,