From 5584cc84c50ca9845c544810fb8331ec8fcdcbc8 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 5 Jan 2024 13:24:34 +0100 Subject: [PATCH] feat: add support for testnets (#437) --- .github/tests/dencun-devnet-12.yaml | 28 +++++++++++++ .github/tests/sepolia-mix.yaml | 16 ++++++++ .github/tests/verkle-gen-devnet-2.yaml | 13 ++++++ .github/workflows/nightly.yml | 5 ++- README.md | 8 ++++ src/cl/lighthouse/lighthouse_launcher.star | 7 +++- src/cl/lodestar/lodestar_launcher.star | 46 ++++++++++++++-------- src/cl/nimbus/nimbus_launcher.star | 8 +++- src/cl/prysm/prysm_launcher.star | 42 ++++++++++++-------- src/cl/teku/teku_launcher.star | 20 +++++++--- src/el/besu/besu_launcher.star | 12 ++++-- src/el/erigon/erigon_launcher.star | 13 ++++-- src/el/ethereumjs/ethereumjs_launcher.star | 13 ++++-- src/el/geth/geth_launcher.star | 30 +++++++++----- src/el/nethermind/nethermind_launcher.star | 15 +++++-- src/el/reth/reth_launcher.star | 17 +++++--- src/package_io/constants.star | 20 ++++++++++ src/participant_network.star | 45 +++++++++++++++------ src/shared_utils/shared_utils.star | 20 ++++++++++ 19 files changed, 294 insertions(+), 84 deletions(-) create mode 100644 .github/tests/dencun-devnet-12.yaml create mode 100644 .github/tests/sepolia-mix.yaml create mode 100644 .github/tests/verkle-gen-devnet-2.yaml diff --git a/.github/tests/dencun-devnet-12.yaml b/.github/tests/dencun-devnet-12.yaml new file mode 100644 index 000000000..72502f846 --- /dev/null +++ b/.github/tests/dencun-devnet-12.yaml @@ -0,0 +1,28 @@ +participants: + - el_client_type: geth + el_client_image: ethpandaops/geth:master + cl_client_type: teku + cl_client_image: ethpandaops/teku:master + - el_client_type: nethermind + el_client_image: ethpandaops/nethermind:master + cl_client_type: prysm + cl_client_image: ethpandaops/prysm:develop,ethpandaops/prysm-validator:develop + - el_client_type: erigon + el_client_image: ethpandaops/erigon:devel + cl_client_type: nimbus + cl_client_image: ethpandaops/nimbus:unstable + - el_client_type: besu + el_client_image: ethpandaops/besu:main + cl_client_type: lighthouse + cl_client_image: ethpandaops/lighthouse:unstable + - el_client_type: reth + el_client_image: ethpandaops/reth:main + cl_client_type: lodestar + cl_client_image: ethpandaops/lodestar:unstable + - el_client_type: ethereumjs + el_client_image: ethpandaops/ethereumjs:master + cl_client_type: teku + cl_client_image: ethpandaops/teku:master +network_params: + network: "dencun-devnet-12" +additional_services: [] diff --git a/.github/tests/sepolia-mix.yaml b/.github/tests/sepolia-mix.yaml new file mode 100644 index 000000000..768ee7e57 --- /dev/null +++ b/.github/tests/sepolia-mix.yaml @@ -0,0 +1,16 @@ +participants: + - el_client_type: geth + cl_client_type: teku + - el_client_type: nethermind + cl_client_type: prysm + - el_client_type: erigon + cl_client_type: lighthouse + - el_client_type: besu + cl_client_type: lighthouse + - el_client_type: reth + cl_client_type: lodestar + - el_client_type: ethereumjs + cl_client_type: nimbus +network_params: + network: "sepolia" +additional_services: [] diff --git a/.github/tests/verkle-gen-devnet-2.yaml b/.github/tests/verkle-gen-devnet-2.yaml new file mode 100644 index 000000000..ded906b52 --- /dev/null +++ b/.github/tests/verkle-gen-devnet-2.yaml @@ -0,0 +1,13 @@ +participants: + - el_client_type: geth + el_client_image: ethpandaops/geth:gballet-kaustinen-with-shapella-fc8f4b9 + cl_client_type: lighthouse + cl_client_image: ethpandaops/lighthouse:verkle-trees-capella-2ffb8a9 + count: 2 + - el_client_type: geth + el_client_image: ethpandaops/geth:gballet-kaustinen-with-shapella-fc8f4b9 + cl_client_type: lodestar + cl_client_image: ethpandaops/lodestar:g11tech-verge-815364b +network_params: + network: verkle-gen-devnet-2 + diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 942d47be1..75b9106ae 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -13,7 +13,8 @@ jobs: steps: - uses: actions/checkout@v4 - id: set-matrix - run: echo "matrix=$(ls ./.github/tests/*.yaml | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT + # List all yaml files in the .github/tests directory, except for the k8s.yaml file + run: echo "matrix=$(ls ./.github/tests/*.yaml | grep -vE 'k8s.yaml$' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT run_with_args: needs: list-yamls strategy: @@ -47,5 +48,5 @@ jobs: DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} with: args: "The nightly test for ${{matrix.file_name}} on ethereum-package has failed find it here ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - + diff --git a/README.md b/README.md index d0218cdfd..0eb40434c 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ This is a [Kurtosis][kurtosis-repo] package that will spin up a private Ethereum testnet over Docker or Kubernetes with multi-client support, Flashbot's `mev-boost` infrastructure for PBS-related testing/validation, and other useful network tools (transaction spammer, monitoring tools, etc). Kurtosis packages are entirely reproducible and composable, so this will work the same way over Docker or Kubernetes, in the cloud or locally on your machine. +You now have the ability to spin up a private Ethereum testnet or public devnet/testnet (e.g. Goerli, Holesky, Sepolia, dencun-devnet-12, verkle-gen-devnet-2 etc) with a single command. This package is designed to be used for testing, validation, and development of Ethereum clients, and is not intended for production use. For more details check network_params.network in the [configuration section](./README.md#configuration). + Specifically, this [package][package-reference] will: 1. Generate Execution Layer (EL) & Consensus Layer (CL) genesis information using [the Ethereum genesis generator](https://github.com/ethpandaops/ethereum-genesis-generator). @@ -272,6 +274,12 @@ network_params: deneb_fork_epoch: 500 electra_fork_epoch: null + # Network name, used to enable syncing of alternative networks + # Defaults to "kurtosis" + # You can sync any public network by setting this to the network name (e.g. "mainnet", "goerli", "sepolia", "holesky") + # You can sync any devnet by setting this to the network name (e.g. "dencun-devnet-12", "verkle-gen-devnet-2") + network: "kurtosis" + # Configuration place for transaction spammer - https:#github.com/MariusVanDerWijden/tx-fuzz tx_spammer_params: # A list of optional extra params that will be passed to the TX Spammer container for modifying its behaviour diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star index 5878343d2..e76d1ddf6 100644 --- a/src/cl/lighthouse/lighthouse_launcher.star +++ b/src/cl/lighthouse/lighthouse_launcher.star @@ -309,7 +309,6 @@ def get_beacon_config( "beacon_node", "--debug-level=" + log_level, "--datadir=" + BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER, - "--testnet-dir=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER, # vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv "--disable-enr-auto-update", "--enr-address=" + PRIVATE_IP_ADDRESS_PLACEHOLDER, @@ -342,6 +341,12 @@ def get_beacon_config( # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^ ] + if network not in constants.PUBLIC_NETWORKS: + cmd.append("--testnet-dir=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER) + else: + cmd.append("--network=" + network) + cmd.append("--checkpoint-sync-url=" + constants.CHECKPOINT_SYNC_URL[network]) + if network == "kurtosis": if boot_cl_client_ctxs != None: cmd.append( diff --git a/src/cl/lodestar/lodestar_launcher.star b/src/cl/lodestar/lodestar_launcher.star index 7d81713d0..9faa4ad08 100644 --- a/src/cl/lodestar/lodestar_launcher.star +++ b/src/cl/lodestar/lodestar_launcher.star @@ -277,12 +277,6 @@ def get_beacon_config( "--port={0}".format(DISCOVERY_PORT_NUM), "--discoveryPort={0}".format(DISCOVERY_PORT_NUM), "--dataDir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, - "--paramsFile=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/config.yaml", - "--genesisStateFile=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/genesis.ssz", "--eth1.depositContractDeployBlock=0", "--network.connectToDiscv5Bootnodes=true", "--discv5=true", @@ -306,21 +300,39 @@ def get_beacon_config( "--metrics.port={0}".format(METRICS_PORT_NUM), # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^ ] - if network == "kurtosis": - if bootnode_contexts != None: + + if network not in constants.PUBLIC_NETWORKS: + cmd.append( + "--paramsFile=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/config.yaml" + ) + cmd.append( + "--genesisStateFile=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/genesis.ssz" + ) + if network == "kurtosis": # kurtosis + if bootnode_contexts != None: + cmd.append( + "--bootnodes=" + + ",".join( + [ + ctx.enr + for ctx in bootnode_contexts[: constants.MAX_ENR_ENTRIES] + ] + ) + ) + else: # devnet cmd.append( "--bootnodes=" - + ",".join( - [ctx.enr for ctx in bootnode_contexts[: constants.MAX_ENR_ENTRIES]] + + shared_utils.get_devnet_enrs_list( + plan, el_cl_genesis_data.files_artifact_uuid ) ) - elif network not in constants.PUBLIC_NETWORKS: - cmd.append( - "--bootnodes=" - + shared_utils.get_devnet_enrs_list( - plan, el_cl_genesis_data.files_artifact_uuid - ) - ) + else: + cmd.append("--network=" + network) + cmd.append("--checkpointSyncUrl=" + constants.CHECKPOINT_SYNC_URL[network]) if len(extra_params) > 0: # this is a repeated, we convert it into Starlark diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star index 5bf613adf..4c1ad4f11 100644 --- a/src/cl/nimbus/nimbus_launcher.star +++ b/src/cl/nimbus/nimbus_launcher.star @@ -51,7 +51,7 @@ VALIDATOR_METRICS_PATH = "/metrics" # Nimbus requires that its data directory already exists (because it expects you to bind-mount it), so we # have to to create it -BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/beacon-data" +BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/nimbus/beacon-data" # Nimbus wants the data dir to have these perms CONSENSUS_DATA_DIR_PERMS_STR = "0700" @@ -313,7 +313,11 @@ def get_beacon_config( "--log-level=" + log_level, "--udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM), "--tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM), - "--network=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER, + "--network={0}".format( + network + if network in constants.PUBLIC_NETWORKS + else constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + ), "--data-dir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--web3-url=" + EXECUTION_ENGINE_ENDPOINT, "--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER, diff --git a/src/cl/prysm/prysm_launcher.star b/src/cl/prysm/prysm_launcher.star index 2a6ecc2a9..19962e5b3 100644 --- a/src/cl/prysm/prysm_launcher.star +++ b/src/cl/prysm/prysm_launcher.star @@ -287,12 +287,6 @@ def get_beacon_config( cmd = [ "--accept-terms-of-use=true", # it's mandatory in order to run the node "--datadir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, - "--chain-config-file=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/config.yaml", - "--genesis-state=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/genesis.ssz", "--execution-endpoint=" + EXECUTION_ENGINE_ENDPOINT, "--rpc-host=0.0.0.0", "--rpc-port={0}".format(RPC_PORT_NUM), @@ -316,18 +310,34 @@ def get_beacon_config( "--monitoring-port={0}".format(BEACON_MONITORING_PORT_NUM) # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^ ] - if network == "kurtosis": - if bootnode_contexts != None: - for ctx in bootnode_contexts[: constants.MAX_ENR_ENTRIES]: - cmd.append("--peer=" + ctx.multiaddr) - cmd.append("--bootstrap-node=" + ctx.enr) - cmd.append("--p2p-static-id=true") - elif network not in constants.PUBLIC_NETWORKS: + if network not in constants.PUBLIC_NETWORKS: + cmd.append("--p2p-static-id=true") cmd.append( - "--bootstrap-node=" - + shared_utils.get_devnet_enr(plan, el_cl_genesis_data.files_artifact_uuid) + "--chain-config-file=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/config.yaml" ) - cmd.append("--p2p-static-id=true") + cmd.append( + "--genesis-state=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/genesis.ssz", + ) + if network == "kurtosis": # Kurtosis + if bootnode_contexts != None: + for ctx in bootnode_contexts[: constants.MAX_ENR_ENTRIES]: + cmd.append("--peer=" + ctx.multiaddr) + cmd.append("--bootstrap-node=" + ctx.enr) + else: # Devnet + cmd.append( + "--bootstrap-node=" + + shared_utils.get_devnet_enr( + plan, el_cl_genesis_data.files_artifact_uuid + ) + ) + else: # Public network + cmd.append("--{}".format(network)) + cmd.append("--genesis-beacon-api-url=" + constants.CHECKPOINT_SYNC_URL[network]) + cmd.append("--checkpoint-sync-url=" + constants.CHECKPOINT_SYNC_URL[network]) if len(extra_params) > 0: # we do the for loop as otherwise its a proto repeated array diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star index 75d7940cb..917ceefb3 100644 --- a/src/cl/teku/teku_launcher.star +++ b/src/cl/teku/teku_launcher.star @@ -302,12 +302,11 @@ def get_beacon_config( cmd = [ "--logging=" + log_level, "--log-destination=CONSOLE", - "--network=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/config.yaml", - "--initial-state=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/genesis.ssz", + "--network={0}".format( + network + if network in constants.PUBLIC_NETWORKS + else constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml" + ), "--data-path=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--data-storage-mode={0}".format( "ARCHIVE" if constants.ARCHIVE_MODE else "PRUNE" @@ -349,6 +348,15 @@ def get_beacon_config( + el_client_context.client_name, ] + if network not in constants.PUBLIC_NETWORKS: + cmd.append( + "--initial-state=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/genesis.ssz" + ) + else: + cmd.append("--checkpoint-sync-url=" + constants.CHECKPOINT_SYNC_URL[network]) + if node_keystore_files != None and not split_mode_enabled: cmd.extend(validator_flags) if network == "kurtosis": diff --git a/src/el/besu/besu_launcher.star b/src/el/besu/besu_launcher.star index 6a8fce143..b7c2b6eca 100644 --- a/src/el/besu/besu_launcher.star +++ b/src/el/besu/besu_launcher.star @@ -152,9 +152,6 @@ def get_config( "besu", "--logging=" + log_level, "--data-path=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, - "--genesis-file=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/besu.json", "--host-allowlist=*", "--rpc-http-enabled=true", "--rpc-http-host=0.0.0.0", @@ -178,6 +175,15 @@ def get_config( "--metrics-host=0.0.0.0", "--metrics-port={0}".format(METRICS_PORT_NUM), ] + if network not in constants.PUBLIC_NETWORKS: + cmd.append( + "--genesis-file=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/besu.json" + ) + else: + cmd.append("--network=" + network) + if network == "kurtosis": if len(existing_el_clients) > 0: cmd.append( diff --git a/src/el/erigon/erigon_launcher.star b/src/el/erigon/erigon_launcher.star index 25489d33b..f5651327e 100644 --- a/src/el/erigon/erigon_launcher.star +++ b/src/el/erigon/erigon_launcher.star @@ -178,7 +178,6 @@ def get_config( "--metrics", "--metrics.addr=0.0.0.0", "--metrics.port={0}".format(METRICS_PORT_NUM), - "--db.size.limit=100GB", ] if network == "kurtosis": @@ -202,6 +201,7 @@ def get_config( ) ) elif network not in constants.PUBLIC_NETWORKS: + cmd.append("--db.size.limit=100GB") cmd.append( "--bootnodes=" + shared_utils.get_devnet_enodes( @@ -219,9 +219,16 @@ def get_config( # this is a repeated, we convert it into Starlark cmd.extend([param for param in extra_params]) - command_arg = [init_datadir_cmd_str, " ".join(cmd)] + if network not in constants.PUBLIC_NETWORKS: + command_arg = [init_datadir_cmd_str, " ".join(cmd)] + + command_arg_str = " && ".join(command_arg) + else: + cmd.append("--chain={0}".format(network)) + cmd.append("--db.size.limit=3TB") + command_arg = cmd + command_arg_str = " ".join(command_arg) - command_arg_str = " && ".join(command_arg) files = { constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file, diff --git a/src/el/ethereumjs/ethereumjs_launcher.star b/src/el/ethereumjs/ethereumjs_launcher.star index 55fad2be3..4e731d758 100644 --- a/src/el/ethereumjs/ethereumjs_launcher.star +++ b/src/el/ethereumjs/ethereumjs_launcher.star @@ -153,9 +153,6 @@ def get_config( persistent, ): cmd = [ - "--gethGenesis=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/genesis.json", "--dataDir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, "--port={0}".format(DISCOVERY_PORT_NUM), "--rpc", @@ -176,6 +173,16 @@ def get_config( "--isSingleNode=true", "--logLevel={0}".format(verbosity_level), ] + + if network not in constants.PUBLIC_NETWORKS: + cmd.append( + "--gethGenesis=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/genesis.json", + ) + else: + cmd.append("--network=" + network) + if network == "kurtosis": if len(existing_el_clients) > 0: cmd.append( diff --git a/src/el/geth/geth_launcher.star b/src/el/geth/geth_launcher.star index f0d9f3c68..00bca8bd9 100644 --- a/src/el/geth/geth_launcher.star +++ b/src/el/geth/geth_launcher.star @@ -168,8 +168,8 @@ def get_config( persistent, ): # TODO: Remove this once electra fork has path based storage scheme implemented - if electra_fork_epoch != None: - if electra_fork_epoch == 0: # verkle-gen + if electra_fork_epoch != None or "verkle" in network: + if electra_fork_epoch == 0 or "verkle-gen" in network: # verkle-gen init_datadir_cmd_str = "geth --datadir={0} --cache.preimages --override.prague={1} init {2}".format( EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, final_genesis_timestamp, @@ -201,18 +201,26 @@ def get_config( "{0}".format( "--state.scheme=path" if electra_fork_epoch == None + and "verkle" not in network and "--builder" not in extra_params and capella_fork_epoch == 0 else "" ), # Override prague fork timestamp for electra fork - "{0}".format("--cache.preimages" if electra_fork_epoch != None else ""), + "{0}".format( + "--cache.preimages" + if electra_fork_epoch != None or "verkle" in network + else "" + ), # Override prague fork timestamp if electra_fork_epoch == 0 "{0}".format( "--override.prague=" + final_genesis_timestamp - if electra_fork_epoch == 0 + if electra_fork_epoch == 0 or "verkle-gen" in network else "" ), + "{0}".format( + "--{}".format(network) if network in constants.PUBLIC_NETWORKS else "" + ), "--verbosity=" + verbosity_level, "--datadir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, "--http", @@ -272,12 +280,14 @@ def get_config( cmd.extend([param for param in extra_params]) cmd_str = " ".join(cmd) - - subcommand_strs = [ - init_datadir_cmd_str, - cmd_str, - ] - command_str = " && ".join(subcommand_strs) + if network not in constants.PUBLIC_NETWORKS: + subcommand_strs = [ + init_datadir_cmd_str, + cmd_str, + ] + command_str = " && ".join(subcommand_strs) + else: + command_str = cmd_str files = { constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, diff --git a/src/el/nethermind/nethermind_launcher.star b/src/el/nethermind/nethermind_launcher.star index 3a5d4e09f..61feaec94 100644 --- a/src/el/nethermind/nethermind_launcher.star +++ b/src/el/nethermind/nethermind_launcher.star @@ -152,11 +152,7 @@ def get_config( cmd = [ "--log=" + log_level, "--datadir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, - "--Init.ChainSpecPath=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/chainspec.json", "--Init.WebSocketsEnabled=true", - "--config=none.cfg", "--JsonRpc.Enabled=true", "--JsonRpc.EnabledModules=net,eth,consensus,subscribe,web3,admin", "--JsonRpc.Host=0.0.0.0", @@ -172,6 +168,17 @@ def get_config( "--Metrics.Enabled=true", "--Metrics.ExposePort={0}".format(METRICS_PORT_NUM), ] + + if network not in constants.PUBLIC_NETWORKS: + cmd.append("--config=none.cfg") + cmd.append( + "--Init.ChainSpecPath=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/chainspec.json" + ) + else: + cmd.append("--config=" + network) + if network == "kurtosis": if len(existing_el_clients) > 0: cmd.append( diff --git a/src/el/reth/reth_launcher.star b/src/el/reth/reth_launcher.star index ef969c2a8..41d1eb49a 100644 --- a/src/el/reth/reth_launcher.star +++ b/src/el/reth/reth_launcher.star @@ -161,7 +161,11 @@ def get_config( "node", "-{0}".format(verbosity_level), "--datadir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, - "--chain=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json", + "--chain={0}".format( + network + if network in constants.PUBLIC_NETWORKS + else constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json" + ), "--http", "--http.port={0}".format(RPC_PORT_NUM), "--http.addr=0.0.0.0", @@ -204,11 +208,14 @@ def get_config( cmd.extend([param for param in extra_params]) cmd_str = " ".join(cmd) + if network not in constants.PUBLIC_NETWORKS: + subcommand_strs = [ + init_datadir_cmd_str, + cmd_str, + ] + else: + subcommand_strs = [cmd_str] - subcommand_strs = [ - init_datadir_cmd_str, - cmd_str, - ] command_str = " && ".join(subcommand_strs) files = { diff --git a/src/package_io/constants.star b/src/package_io/constants.star index 9d59af240..08824c874 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -59,3 +59,23 @@ PUBLIC_NETWORKS = ( "sepolia", "holesky", ) +CHECKPOINT_SYNC_URL = { + "mainnet": "https://beaconstate.info", + "goerli": "https://checkpoint-sync.goerli.ethpandaops.io", + "sepolia": "https://checkpoint-sync.sepolia.ethpandaops.io", + "holesky": "https://checkpoint-sync.holesky.ethpandaops.io", +} + +GENESIS_VALIDATORS_ROOT = { + "mainnet": "0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95", + "goerli": "0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb", + "sepolia": "0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078", + "holesky": "0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1", +} + +GENESIS_TIME = { + "mainnet": 1606824023, + "goerli": 1616508000, + "sepolia": 1655733600, + "holesky": 1695902400, +} diff --git a/src/participant_network.star b/src/participant_network.star index 61d3335e7..2c854938a 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -65,6 +65,7 @@ def launch_participant_network( ): num_participants = len(participants) if network_params.network == "kurtosis": + # We are running a kurtosis network plan.print("Generating cl validator key stores") validator_data = None if not parallel_keystore_generation: @@ -149,14 +150,23 @@ def launch_participant_network( network_params.deneb_fork_epoch, network_params.electra_fork_epoch, ) + elif network_params.network in constants.PUBLIC_NETWORKS: + # We are running a public network + dummy = plan.run_sh( + run="mkdir /network-configs", + store=[StoreSpec(src="/network-configs/", name="el_cl_genesis_data")], + ) + el_cl_data = el_cl_genesis_data.new_el_cl_genesis_data( + dummy.files_artifacts[0], + constants.GENESIS_VALIDATORS_ROOT[network_params.network], + ) + final_genesis_timestamp = constants.GENESIS_TIME[network_params.network] + validator_data = None else: - # split up the name from dencun-devnet-12 to dencun and devnet-12 - devnet_name = network_params.network.split("-")[0] - devnet_number = network_params.network.split("-")[-1] + # We are running a devnet + url = calculate_devnet_url(network_params.network) el_cl_genesis_uuid = plan.upload_files( - src="github.com/ethpandaops/{0}-devnets/network-configs/devnet-{1}".format( - devnet_name, devnet_number - ), + src=url, name="el_cl_genesis", ) el_cl_genesis_data_uuid = plan.run_sh( @@ -164,11 +174,14 @@ def launch_participant_network( store=[StoreSpec(src="/network-configs/", name="el_cl_genesis_data")], files={"/opt": el_cl_genesis_uuid}, ) + genesis_validators_root = read_file(url + "/genesis_validators_root.txt") el_cl_data = el_cl_genesis_data.new_el_cl_genesis_data( el_cl_genesis_data_uuid.files_artifacts[0], - "0", + genesis_validators_root, + ) + final_genesis_timestamp = shared_utils.read_genesis_timestamp_from_config( + plan, el_cl_genesis_uuid ) - final_genesis_timestamp = 0 validator_data = None el_launchers = { @@ -527,10 +540,6 @@ def launch_participant_network( ) -def zfill_custom(value, width): - return ("0" * (width - len(str(value)))) + str(value) - - # this is a python procedure so that Kurtosis can do idempotent runs # time.now() runs everytime bringing non determinism # note that the timestamp it returns is a string @@ -546,3 +555,15 @@ print(int(time.time()+padding), end="") store=[StoreSpec(src="/tmp", name="final-genesis-timestamp")], ) return result.output + + +def calculate_devnet_url(network): + network_parts = network.split("-devnet-", 1) + devnet_name, devnet_number = network_parts[0], network_parts[1] + devnet_category = devnet_name.split("-")[0] + devnet_subname = ( + devnet_name.split("-")[1] + "-" if len(devnet_name.split("-")) > 1 else "" + ) + return "github.com/ethpandaops/{0}-devnets/network-configs/{1}devnet-{2}".format( + devnet_category, devnet_subname, devnet_number + ) diff --git a/src/shared_utils/shared_utils.star b/src/shared_utils/shared_utils.star index 656bea513..43694e310 100644 --- a/src/shared_utils/shared_utils.star +++ b/src/shared_utils/shared_utils.star @@ -121,3 +121,23 @@ with open("/network-configs/network-configs/bootstrap_nodes.txt") as bootnode_fi """, ) return enr_items.output + + +def read_genesis_timestamp_from_config(plan, filename): + value = plan.run_python( + files={constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: filename}, + wait=None, + packages=["PyYAML"], + run=""" +import yaml +with open("/network-configs/config.yaml", "r") as f: + yaml_data = yaml.safe_load(f) + +# Get values from the YAML content +min_genesis_time = int(yaml_data.get("MIN_GENESIS_TIME", 0)) +genesis_delay = int(yaml_data.get("GENESIS_DELAY", 0)) + +print(int(min_genesis_time + genesis_delay), end="") + """, + ) + return value.output