Merge branch 'zkevm-2.60' into arpit/update-zero-file

.github/workflows/ci_zkevm.yml

@@ -23,9 +23,9 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ ubuntu-22.04, macos-14 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-22.04, macos-14-xlarge ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
-    timeout-minutes: ${{ matrix.os == 'macos-14' && 40 || 30 }}
+    timeout-minutes: ${{ matrix.os == 'macos-14-xlarge' && 40 || 30 }}
 
     steps:
       - uses: actions/checkout@v3

Dockerfile

@@ -84,7 +84,7 @@ COPY --from=builder /app/build/bin/sentry /usr/local/bin/sentry
 COPY --from=builder /app/build/bin/state /usr/local/bin/state
 COPY --from=builder /app/build/bin/txpool /usr/local/bin/txpool
 COPY --from=builder /app/build/bin/verkle /usr/local/bin/verkle
-
+COPY --from=builder /app/build/bin/acl /usr/local/bin/acl
 
 EXPOSE 8545 \
        8551 \

Dockerfile.debian

@@ -75,6 +75,7 @@ COPY --from=builder /app/build/bin/state /usr/local/bin/state
 COPY --from=builder /app/build/bin/txpool /usr/local/bin/txpool
 COPY --from=builder /app/build/bin/verkle /usr/local/bin/verkle
 COPY --from=builder /app/build/bin/caplin /usr/local/bin/caplin
+COPY --from=builder /app/build/bin/acl /usr/local/bin/acl
 
 COPY --from=builder /go/pkg/mod /go/pkg/mod
 

Makefile

@@ -167,7 +167,7 @@ test-erigon-ext:
 
 ## test:                              run unit tests with a 100s timeout
 test:
-	$(GOTEST) --timeout 200s
+	$(GOTEST) --timeout 10m
 
 test3:
 	$(GOTEST) --timeout 200s -tags $(BUILD_TAGS),erigon3

cmd/cdk-erigon/main.go

@@ -110,13 +110,17 @@ func setFlagsFromConfigFile(ctx *cli.Context, filePath string) error {
 				for i, v := range sliceInterface {
 					s[i] = fmt.Sprintf("%v", v)
 				}
-				err := ctx.Set(key, strings.Join(s, ","))
-				if err != nil {
+				if err := ctx.Set(key, strings.Join(s, ",")); err != nil {
+					if deprecatedFlag, found := erigoncli.DeprecatedFlags[key]; found {
+						return fmt.Errorf("failed setting %s flag Flag is deprecated, use %s instead", key, deprecatedFlag)
+					}
 					return fmt.Errorf("failed setting %s flag with values=%s error=%s", key, s, err)
 				}
 			} else {
-				err := ctx.Set(key, fmt.Sprintf("%v", value))
-				if err != nil {
+				if err := ctx.Set(key, fmt.Sprintf("%v", value)); err != nil {
+					if deprecatedFlag, found := erigoncli.DeprecatedFlags[key]; found {
+						return fmt.Errorf("failed setting %s flag Flag is deprecated, use %s instead", key, deprecatedFlag)
+					}
 					return fmt.Errorf("failed setting %s flag with value=%v error=%s", key, value, err)
 				}
 			}

cmd/integration/commands/flags.go

@@ -35,6 +35,7 @@ var (
 	unwindTypes                             []string
 	chain                                   string // Which chain to use (mainnet, goerli, sepolia, etc.)
 	outputCsvFile                           string
+	config                                  string
 
 	commitmentMode string
 	commitmentTrie string
@@ -53,7 +54,7 @@ func must(err error) {
 }
 
 func withConfig(cmd *cobra.Command) {
-	cmd.Flags().String("config", "", "yaml/toml config file location")
+	cmd.Flags().StringVar(&config, "config", "", "yaml/toml config file location")
 }
 
 func withMining(cmd *cobra.Command) {

cmd/integration/commands/stage_stages_zkevm.go

@@ -9,11 +9,8 @@ import (
 	common2 "github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon-lib/wrap"
-	"github.com/ledgerwatch/erigon/core"
-	"github.com/ledgerwatch/erigon/eth/ethconfig"
 	"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
 	smtdb "github.com/ledgerwatch/erigon/smt/pkg/db"
-	erigoncli "github.com/ledgerwatch/erigon/turbo/cli"
 	"github.com/ledgerwatch/erigon/zk/hermez_db"
 	"github.com/ledgerwatch/log/v3"
 	"github.com/spf13/cobra"
@@ -29,9 +26,6 @@ state_stages_zkevm --datadir=/datadirs/hermez-mainnet --unwind-batch-no=2 --chai
 	Example: "go run ./cmd/integration state_stages_zkevm --config=... --verbosity=3 --unwind-batch-no=100",
 	Run: func(cmd *cobra.Command, args []string) {
 		ctx, _ := common2.RootContext()
-		ethConfig := &ethconfig.Defaults
-		ethConfig.Genesis = core.GenesisBlockByChainName(chain)
-		erigoncli.ApplyFlagsForEthConfigCobra(cmd.Flags(), ethConfig)
 		logger := log.New()
 		db, err := openDB(dbCfg(kv.ChainDB, chaindata), true, logger)
 		if err != nil {

cmd/integration/commands/stages_zkevm.go

@@ -2,13 +2,20 @@ package commands
 
 import (
 	"context"
+	"encoding/json"
+	"math/big"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
 
 	"github.com/c2h5oh/datasize"
 	chain3 "github.com/ledgerwatch/erigon-lib/chain"
 	"github.com/ledgerwatch/erigon-lib/common/datadir"
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
 	"github.com/ledgerwatch/erigon/cmd/hack/tool/fromdb"
+	"github.com/ledgerwatch/erigon/cmd/utils"
 	"github.com/ledgerwatch/erigon/consensus"
 	"github.com/ledgerwatch/erigon/core"
 	"github.com/ledgerwatch/erigon/core/types"
@@ -17,6 +24,7 @@ import (
 	"github.com/ledgerwatch/erigon/eth/stagedsync"
 	"github.com/ledgerwatch/erigon/p2p/sentry"
 	"github.com/ledgerwatch/erigon/p2p/sentry/sentry_multi_client"
+	"github.com/ledgerwatch/erigon/params"
 	"github.com/ledgerwatch/erigon/turbo/shards"
 	stages2 "github.com/ledgerwatch/erigon/turbo/stages"
 	"github.com/ledgerwatch/erigon/zk/sequencer"
@@ -29,7 +37,36 @@ func newSyncZk(ctx context.Context, db kv.RwDB) (consensus.Engine, *vm.Config, *
 
 	vmConfig := &vm.Config{}
 
-	genesis := core.GenesisBlockByChainName(chain)
+	var genesis *types.Genesis
+
+	if strings.HasPrefix(chain, "dynamic") {
+		if config == "" {
+			panic("Config file is required for dynamic chain")
+		}
+
+		params.DynamicChainConfigPath = filepath.Dir(config)
+		genesis = core.GenesisBlockByChainName(chain)
+		filename := path.Join(params.DynamicChainConfigPath, chain+"-conf.json")
+
+		dConf := utils.DynamicConfig{}
+
+		if _, err := os.Stat(filename); err == nil {
+			dConfBytes, err := os.ReadFile(filename)
+			if err != nil {
+				panic(err)
+			}
+			if err := json.Unmarshal(dConfBytes, &dConf); err != nil {
+				panic(err)
+			}
+		}
+
+		genesis.Timestamp = dConf.Timestamp
+		genesis.GasLimit = dConf.GasLimit
+		genesis.Difficulty = big.NewInt(dConf.Difficulty)
+	} else {
+		genesis = core.GenesisBlockByChainName(chain)
+	}
+
 	chainConfig, genesisBlock, genesisErr := core.CommitGenesisBlock(db, genesis, "", log.New())
 	if _, ok := genesisErr.(*chain3.ConfigCompatError); genesisErr != nil && !ok {
 		panic(genesisErr)

cmd/rpcdaemon/cli/config.go

@@ -93,7 +93,6 @@ func RootCommand() (*cobra.Command, *httpcfg.HttpCfg) {
 	rootCmd.PersistentFlags().BoolVar(&cfg.GraphQLEnabled, "graphql", false, "enables graphql endpoint (disabled by default)")
 	rootCmd.PersistentFlags().Uint64Var(&cfg.Gascap, "rpc.gascap", 50_000_000, "Sets a cap on gas that can be used in eth_call/estimateGas")
 	rootCmd.PersistentFlags().Uint64Var(&cfg.MaxTraces, "trace.maxtraces", 200, "Sets a limit on traces that can be returned in trace_filter")
-
 	rootCmd.PersistentFlags().StringVar(&cfg.RpcAllowListFilePath, utils.RpcAccessListFlag.Name, "", "Specify granular (method-by-method) API allowlist")
 	rootCmd.PersistentFlags().UintVar(&cfg.RpcBatchConcurrency, utils.RpcBatchConcurrencyFlag.Name, 2, utils.RpcBatchConcurrencyFlag.Usage)
 	rootCmd.PersistentFlags().BoolVar(&cfg.RpcStreamingDisable, utils.RpcStreamingDisableFlag.Name, false, utils.RpcStreamingDisableFlag.Usage)
@@ -124,6 +123,9 @@ func RootCommand() (*cobra.Command, *httpcfg.HttpCfg) {
 	rootCmd.PersistentFlags().BoolVar(&cfg.HttpCompression, "http.compression", true, "Disable http compression")
 	rootCmd.PersistentFlags().BoolVar(&cfg.WebsocketEnabled, "ws", false, "Enable Websockets - Same port as HTTP[S]")
 	rootCmd.PersistentFlags().BoolVar(&cfg.WebsocketCompression, "ws.compression", false, "Enable Websocket compression (RFC 7692)")
+	rootCmd.PersistentFlags().StringVar(&cfg.WebSocketListenAddress, "ws.addr", nodecfg.DefaultHTTPHost, "Websocket server listening interface")
+	rootCmd.PersistentFlags().IntVar(&cfg.WebSocketPort, "ws.port", nodecfg.DefaultHTTPPort, "Websocket server listening port")
+	rootCmd.PersistentFlags().StringSliceVar(&cfg.WebsocketCORSDomain, "ws.corsdomain", []string{}, "Comma separated list of domains from which to accept cross origin requests (browser enforced)")
 
 	rootCmd.PersistentFlags().BoolVar(&cfg.HttpsServerEnabled, "https.enabled", false, "enable http server")
 	rootCmd.PersistentFlags().StringVar(&cfg.HttpsListenAddress, "https.addr", nodecfg.DefaultHTTPHost, "rpc HTTPS server listening interface")
@@ -630,37 +632,14 @@ func startRegularRpcServer(ctx context.Context, cfg *httpcfg.HttpCfg, rpcAPI []r
 	}
 
 	httpHandler := node.NewHTTPHandlerStack(srv, cfg.HttpCORSDomain, cfg.HttpVirtualHost, cfg.HttpCompression)
-	var wsHandler http.Handler
-	if cfg.WebsocketEnabled {
-		wsHandler = srv.WebsocketHandler([]string{"*"}, nil, cfg.WebsocketCompression, logger)
-	}
+
 	graphQLHandler := graphql.CreateHandler(defaultAPIList)
-	apiHandler, err := createHandler(cfg, defaultAPIList, httpHandler, wsHandler, graphQLHandler, nil)
+
+	apiHandler, err := createHandler(*cfg, defaultAPIList, httpHandler, nil, graphQLHandler, nil)
 	if err != nil {
 		return err
 	}
 
-	// Separate Websocket handler if websocket port flag specified
-	if cfg.WebsocketEnabled && cfg.WebsocketPort != cfg.HttpPort {
-		wsEndpoint := fmt.Sprintf("tcp://%s:%d", cfg.HttpListenAddress, cfg.WebsocketPort)
-		wsApiHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if isWebsocket(r) {
-				wsHandler.ServeHTTP(w, r)
-			}
-		})
-		wsListener, wsAddr, err := node.StartHTTPEndpoint(wsEndpoint, &node.HttpEndpointConfig{Timeouts: cfg.HTTPTimeouts}, wsApiHandler)
-		if err != nil {
-			return fmt.Errorf("could not start separate Websocket RPC api at port %d: %w", cfg.WebsocketPort, err)
-		}
-		info = append(info, "websocket.url", wsAddr)
-		defer func() {
-			shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-			defer cancel()
-			_ = wsListener.Shutdown(shutdownCtx)
-			logger.Info("HTTP endpoint closed", "url", wsAddr)
-		}()
-	}
-
 	if cfg.HttpServerEnabled {
 		httpEndpoint := fmt.Sprintf("tcp://%s:%d", cfg.HttpListenAddress, cfg.HttpPort)
 		if cfg.HttpURL != "" {
@@ -680,6 +659,7 @@ func startRegularRpcServer(ctx context.Context, cfg *httpcfg.HttpCfg, rpcAPI []r
 			logger.Info("HTTP endpoint closed", "url", httpAddr)
 		}()
 	}
+
 	if cfg.HttpsURL != "" {
 		cfg.HttpsServerEnabled = true
 	}
@@ -740,7 +720,72 @@ func startRegularRpcServer(ctx context.Context, cfg *httpcfg.HttpCfg, rpcAPI []r
 		}()
 	}
 
-	logger.Info("JsonRpc endpoint opened", info...)
+	log.Info("HTTP endpoint opened", info...)
+
+	if cfg.WebsocketEnabled {
+		wsSrv := rpc.NewServer(cfg.RpcBatchConcurrency, cfg.TraceRequests, cfg.RpcStreamingDisable, false, logger, cfg.RPCSlowLogThreshold)
+
+		allowListForRPC, err := parseAllowListForRPC(cfg.RpcAllowListFilePath)
+		if err != nil {
+			return err
+		}
+
+		var wsApiFlags []string
+		for _, flag := range cfg.WebSocketApi {
+			if flag != "engine" {
+				wsApiFlags = append(wsApiFlags, flag)
+			}
+		}
+
+		if err := node.RegisterApisFromWhitelist(defaultAPIList, wsApiFlags, wsSrv, false, logger); err != nil {
+			return fmt.Errorf("could not start register WS apis: %w", err)
+		}
+		wsSrv.SetAllowList(allowListForRPC)
+
+		wsSrv.SetBatchLimit(cfg.BatchLimit)
+
+		var defaultAPIList []rpc.API
+
+		for _, api := range rpcAPI {
+			if api.Namespace != "engine" {
+				defaultAPIList = append(defaultAPIList, api)
+			}
+		}
+
+		var apiFlags []string
+		for _, flag := range cfg.API {
+			if flag != "engine" {
+				apiFlags = append(apiFlags, flag)
+			}
+		}
+
+		if err := node.RegisterApisFromWhitelist(defaultAPIList, apiFlags, wsSrv, false, logger); err != nil {
+			return fmt.Errorf("could not start register RPC apis: %w", err)
+		}
+
+		wsEndpoint := fmt.Sprintf("tcp://%s:%d", cfg.WebSocketListenAddress, cfg.WebSocketPort)
+
+		wsHttpHandler := wsSrv.WebsocketHandler(cfg.WebsocketCORSDomain, nil, cfg.WebsocketCompression, logger)
+		wsHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			wsHttpHandler.ServeHTTP(w, r)
+		})
+
+		wsListener, wsHttpAddr, err := node.StartHTTPEndpoint(wsEndpoint, &node.HttpEndpointConfig{
+			Timeouts: cfg.HTTPTimeouts,
+		}, wsHandler)
+		if err != nil {
+			return fmt.Errorf("could not start ws RPC api: %w", err)
+		}
+
+		defer func() {
+			wsSrv.Stop()
+			shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			_ = wsListener.Shutdown(shutdownCtx)
+			log.Info("WS endpoint closed", "url", wsHttpAddr)
+		}()
+	}
+
 	<-ctx.Done()
 	logger.Info("Exiting...")
 	return nil
@@ -816,7 +861,7 @@ func ObtainJWTSecret(cfg *httpcfg.HttpCfg, logger log.Logger) ([]byte, error) {
 	return jwtSecret, nil
 }
 
-func createHandler(cfg *httpcfg.HttpCfg, apiList []rpc.API, httpHandler http.Handler, wsHandler http.Handler, graphQLHandler http.Handler, jwtSecret []byte) (http.Handler, error) {
+func createHandler(cfg httpcfg.HttpCfg, apiList []rpc.API, httpHandler, wsHandler, graphQLHandler http.Handler, jwtSecret []byte) (http.Handler, error) {
 	var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if cfg.GraphQLEnabled && graphql.ProcessGraphQLcheckIfNeeded(graphQLHandler, w, r) {
 			return
@@ -861,7 +906,7 @@ func createEngineListener(cfg *httpcfg.HttpCfg, engineApi []rpc.API, logger log.
 
 	graphQLHandler := graphql.CreateHandler(engineApi)
 
-	engineApiHandler, err := createHandler(cfg, engineApi, engineHttpHandler, wsHandler, graphQLHandler, jwtSecret)
+	engineApiHandler, err := createHandler(*cfg, engineApi, engineHttpHandler, wsHandler, graphQLHandler, jwtSecret)
 	if err != nil {
 		return nil, nil, "", err
 	}

cmd/rpcdaemon/cli/httpcfg/http_cfg.go

@@ -48,6 +48,10 @@ type HttpCfg struct {
 	WebsocketEnabled                  bool
 	WebsocketCompression              bool
 	WebsocketSubscribeLogsChannelSize int
+	WebSocketListenAddress            string
+	WebSocketPort                     int
+	WebsocketCORSDomain               []string
+	WebSocketApi                      []string
 	RpcAllowListFilePath              string
 	RpcBatchConcurrency               uint
 	RpcStreamingDisable               bool