From 8e2e1a38d301b11ccaa7818c1b8407c42778f07b Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Tue, 13 Feb 2024 00:53:32 +0100 Subject: [PATCH 01/11] Separate validator clients from CL clients --- .github/tests/mix-persistence-k8s.yaml | 4 +- .github/tests/mix-persistence.yaml | 4 +- .github/tests/node-selectors.yaml | 4 +- .github/tests/split-nimbus.yaml | 12 +- .github/tests/split-teku.yaml | 12 +- .github/tests/tolerations.yaml | 6 +- README.md | 29 ++- main.star | 3 + src/cl/cl_client_context.star | 2 - src/cl/lighthouse/lighthouse_launcher.star | 170 +--------------- src/cl/lodestar/lodestar_launcher.star | 147 +------------- src/cl/nimbus/nimbus_launcher.star | 170 +--------------- src/cl/prysm/prysm_launcher.star | 189 +----------------- src/cl/teku/teku_launcher.star | 189 +----------------- src/package_io/constants.star | 8 + src/package_io/input_parser.star | 57 ++++-- src/participant.star | 4 + src/participant_network.star | 93 +++++++-- src/prometheus/prometheus_launcher.star | 56 +++--- src/validator_client/lighthouse.star | 101 ++++++++++ src/validator_client/lodestar.star | 95 +++++++++ src/validator_client/nimbus.star | 77 +++++++ src/validator_client/prysm.star | 88 ++++++++ src/validator_client/shared.star | 16 ++ src/validator_client/teku.star | 84 ++++++++ .../validator_client_context.star | 10 + .../validator_client_launcher.star | 176 ++++++++++++++++ 27 files changed, 865 insertions(+), 941 deletions(-) create mode 100644 src/validator_client/lighthouse.star create mode 100644 src/validator_client/lodestar.star create mode 100644 src/validator_client/nimbus.star create mode 100644 src/validator_client/prysm.star create mode 100644 src/validator_client/shared.star create mode 100644 src/validator_client/teku.star create mode 100644 src/validator_client/validator_client_context.star create mode 100644 src/validator_client/validator_client_launcher.star diff --git a/.github/tests/mix-persistence-k8s.yaml b/.github/tests/mix-persistence-k8s.yaml index 252af09b7..4c6d8443e 100644 --- a/.github/tests/mix-persistence-k8s.yaml +++ b/.github/tests/mix-persistence-k8s.yaml @@ -1,13 +1,13 @@ participants: - el_client_type: geth cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true - el_client_type: nethermind cl_client_type: prysm - el_client_type: erigon cl_client_type: nimbus cl_client_image: ethpandaops/nimbus:unstable - cl_split_mode_enabled: true + use_separate_validator_client: true - el_client_type: besu cl_client_type: lighthouse - el_client_type: reth diff --git a/.github/tests/mix-persistence.yaml b/.github/tests/mix-persistence.yaml index 252af09b7..4c6d8443e 100644 --- a/.github/tests/mix-persistence.yaml +++ b/.github/tests/mix-persistence.yaml @@ -1,13 +1,13 @@ participants: - el_client_type: geth cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true - el_client_type: nethermind cl_client_type: prysm - el_client_type: erigon cl_client_type: nimbus cl_client_image: ethpandaops/nimbus:unstable - cl_split_mode_enabled: true + use_separate_validator_client: true - el_client_type: besu cl_client_type: lighthouse - el_client_type: reth diff --git a/.github/tests/node-selectors.yaml b/.github/tests/node-selectors.yaml index a011f73dc..fdd34e48e 100644 --- a/.github/tests/node-selectors.yaml +++ b/.github/tests/node-selectors.yaml @@ -1,13 +1,13 @@ participants: - el_client_type: reth cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true node_selectors: { "kubernetes.io/hostname": testing-1, } - el_client_type: reth cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true global_node_selectors: { "kubernetes.io/hostname": testing-2, } diff --git a/.github/tests/split-nimbus.yaml b/.github/tests/split-nimbus.yaml index d44805ff4..21720cf85 100644 --- a/.github/tests/split-nimbus.yaml +++ b/.github/tests/split-nimbus.yaml @@ -2,26 +2,26 @@ participants: - el_client_type: geth cl_client_type: nimbus cl_client_image: ethpandaops/nimbus:unstable - cl_split_mode_enabled: true + use_separate_validator_client: true validator_count: 0 - el_client_type: nethermind cl_client_type: nimbus - cl_split_mode_enabled: true + use_separate_validator_client: true cl_client_image: ethpandaops/nimbus:unstable - el_client_type: erigon cl_client_type: nimbus - cl_split_mode_enabled: true + use_separate_validator_client: true cl_client_image: ethpandaops/nimbus:unstable - el_client_type: besu cl_client_type: nimbus - cl_split_mode_enabled: true + use_separate_validator_client: true cl_client_image: ethpandaops/nimbus:unstable - el_client_type: reth cl_client_type: nimbus - cl_split_mode_enabled: true + use_separate_validator_client: true cl_client_image: ethpandaops/nimbus:unstable - el_client_type: ethereumjs cl_client_type: nimbus - cl_split_mode_enabled: true + use_separate_validator_client: true cl_client_image: ethpandaops/nimbus:unstable additional_services: [] diff --git a/.github/tests/split-teku.yaml b/.github/tests/split-teku.yaml index 8b99808c6..c5dbe4bae 100644 --- a/.github/tests/split-teku.yaml +++ b/.github/tests/split-teku.yaml @@ -1,21 +1,21 @@ participants: - el_client_type: geth cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true validator_count: 0 - el_client_type: nethermind cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true - el_client_type: erigon cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true - el_client_type: besu cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true - el_client_type: reth cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true - el_client_type: ethereumjs cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true additional_services: [] diff --git a/.github/tests/tolerations.yaml b/.github/tests/tolerations.yaml index 140938834..5635d2f6f 100644 --- a/.github/tests/tolerations.yaml +++ b/.github/tests/tolerations.yaml @@ -1,7 +1,7 @@ participants: - el_client_type: reth cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true cl_tolerations: - key: "node-role.kubernetes.io/master1" operator: "Exists" @@ -19,14 +19,14 @@ participants: effect: "NoSchedule" - el_client_type: reth cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true tolerations: - key: "node-role.kubernetes.io/master5" operator: "Exists" effect: "NoSchedule" - el_client_type: reth cl_client_type: teku - cl_split_mode_enabled: true + use_separate_validator_client: true additional_services: - dora global_tolerations: diff --git a/README.md b/README.md index 3b1773563..a2f6ba402 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ When running on a public testnet using a cloud provider's Kubernetes cluster, th 1. State Growth: The growth of the state might be faster than anticipated. This could potentially lead to issues if the default parameters become insufficient over time. It's important to monitor state growth and adjust parameters as necessary. -2. Persistent Storage Speed: Most cloud providers provision their Kubernetes clusters with relatively slow persistent storage by default. This can cause performance issues, particularly with Ethereum Light (EL) clients. +2. Persistent Storage Speed: Most cloud providers provision their Kubernetes clusters with relatively slow persistent storage by default. This can cause performance issues, particularly with Execution Layer (EL) clients. 3. Network Syncing: The disk speed provided by cloud providers may not be sufficient to sync with networks that have high demands, such as the mainnet. This could lead to syncing issues and delays. @@ -203,7 +203,7 @@ participants: # Valid values are nimbus, lighthouse, lodestar, teku, and prysm cl_client_type: lighthouse - # The Docker image that should be used for the EL client; leave blank to use the default for the client type + # The Docker image that should be used for the CL client; leave blank to use the default for the client type # Defaults by client (note that Prysm is different in that it requires two images - a Beacon and a validator - separated by a comma): # - lighthouse: sigp/lighthouse:latest # - teku: consensys/teku:latest @@ -212,18 +212,31 @@ participants: # - lodestar: chainsafe/lodestar:next cl_client_image: "" - # The log level string that this participant's EL client should log at + # The log level string that this participant's CL client should log at # If this is emptystring then the global `logLevel` parameter's value will be translated into a string appropriate for the client (e.g. if # global `logLevel` = `info` then Teku would receive `INFO`, Prysm would receive `info`, etc.) # If this is not emptystring, then this value will override the global `logLevel` setting to allow for fine-grained control # over a specific participant's logging cl_client_log_level: "" - # A list of optional extra params that will be passed to the CL to run separate Beacon and validator nodes - # Only possible for nimbus or teku - # Please note that in order to get it to work with Nimbus, you have to use `ethpandaops/nimbus:unstable` as the image (default upstream image does not yet support this out of the box) - # Defaults to false - cl_split_mode_enabled: false + # Whether to use a separate validator client attached to the CL client. + # Defaults to false for clients that can run both in one process (Teku, Nimbus) + use_separate_validator_client: true/false + + # The type of validator client that should be used + # Valid values are nimbus, lighthouse, lodestar, teku, and prysm + # ( The prysm validator only works with a prysm CL client ) + # Defaults to matching the chosen CL client (cl_client_type) + validator_client_type: "" + + # The Docker image that should be used for the separate validator client + # Defaults by client: + # - lighthouse: sigp/lighthouse:latest + # - lodestar: chainsafe/lodestar:latest + # - nimbus: statusim/nimbus-validator-client:multiarch-latest + # - prysm: gcr.io/prysmaticlabs/prysm/validator:latest + # - teku: consensys/teku:latest + validator_client_image: "" # Persistent storage size for the CL client container (in MB) # Defaults to 0, which means that the default size for the client will be used diff --git a/main.star b/main.star index b1ebc4dbf..68bac3681 100644 --- a/main.star +++ b/main.star @@ -109,11 +109,13 @@ def run(plan, args={}): all_el_client_contexts = [] all_cl_client_contexts = [] + all_validator_client_contexts = [] all_ethereum_metrics_exporter_contexts = [] all_xatu_sentry_contexts = [] for participant in all_participants: all_el_client_contexts.append(participant.el_client_context) all_cl_client_contexts.append(participant.cl_client_context) + all_validator_client_contexts.append(participant.validator_client_context) all_ethereum_metrics_exporter_contexts.append( participant.ethereum_metrics_exporter_context ) @@ -426,6 +428,7 @@ def run(plan, args={}): plan, all_el_client_contexts, all_cl_client_contexts, + all_validator_client_contexts, prometheus_additional_metrics_jobs, all_ethereum_metrics_exporter_contexts, all_xatu_sentry_contexts, diff --git a/src/cl/cl_client_context.star b/src/cl/cl_client_context.star index a0e198014..0aeb9f35c 100644 --- a/src/cl/cl_client_context.star +++ b/src/cl/cl_client_context.star @@ -5,7 +5,6 @@ def new_cl_client_context( http_port_num, cl_nodes_metrics_info, beacon_service_name, - validator_service_name="", multiaddr="", peer_id="", snooper_enabled=False, @@ -19,7 +18,6 @@ def new_cl_client_context( http_port_num=http_port_num, cl_nodes_metrics_info=cl_nodes_metrics_info, beacon_service_name=beacon_service_name, - validator_service_name=validator_service_name, multiaddr=multiaddr, peer_id=peer_id, snooper_enabled=snooper_enabled, diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star index c22694847..f0cead5ff 100644 --- a/src/cl/lighthouse/lighthouse_launcher.star +++ b/src/cl/lighthouse/lighthouse_launcher.star @@ -30,22 +30,7 @@ BEACON_METRICS_PORT_NUM = 5054 BEACON_MIN_CPU = 50 BEACON_MIN_MEMORY = 256 -# ---------------------------------- Validator client ------------------------------------- -VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/data/lighthouse/validator-keys" -VALIDATOR_HTTP_PORT_ID = "http" -VALIDATOR_METRICS_PORT_ID = "metrics" -VALIDATOR_HTTP_PORT_NUM = 5042 -VALIDATOR_METRICS_PORT_NUM = 5064 -VALIDATOR_HTTP_PORT_WAIT_DISABLED = None - METRICS_PATH = "/metrics" -VALIDATOR_SUFFIX_SERVICE_NAME = "validator" - -# The min/max CPU/memory that the validator node can use -VALIDATOR_MIN_CPU = 50 -VALIDATOR_MAX_CPU = 300 -VALIDATOR_MIN_MEMORY = 128 -VALIDATOR_MAX_MEMORY = 512 PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" @@ -68,20 +53,6 @@ BEACON_USED_PORTS = { ), } -VALIDATOR_USED_PORTS = { - VALIDATOR_HTTP_PORT_ID: shared_utils.new_port_spec( - VALIDATOR_HTTP_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.NOT_PROVIDED_APPLICATION_PROTOCOL, - VALIDATOR_HTTP_PORT_WAIT_DISABLED, - ), - VALIDATOR_METRICS_PORT_ID: shared_utils.new_port_spec( - VALIDATOR_METRICS_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), -} - VERBOSITY_LEVELS = { constants.GLOBAL_CLIENT_LOG_LEVEL.error: "error", constants.GLOBAL_CLIENT_LOG_LEVEL.warn: "warn", @@ -105,31 +76,21 @@ def launch( bn_max_cpu, bn_min_mem, bn_max_mem, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, snooper_enabled, snooper_engine_context, blobber_enabled, blobber_extra_params, extra_beacon_params, - extra_validator_params, extra_beacon_labels, - extra_validator_labels, persistent, cl_volume_size, cl_tolerations, - validator_tolerations, participant_tolerations, global_tolerations, node_selectors, - split_mode_enabled=False, + use_separate_validator_client=True, ): beacon_service_name = "{0}".format(service_name) - validator_service_name = "{0}-{1}".format( - service_name, VALIDATOR_SUFFIX_SERVICE_NAME - ) log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -211,37 +172,6 @@ def launch( ) beacon_http_url = blobber_http_url - # Launch validator node if we have a keystore - validator_service = None - if node_keystore_files != None: - v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU - v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU - v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY - v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY - tolerations = input_parser.get_client_tolerations( - validator_tolerations, participant_tolerations, global_tolerations - ) - validator_config = get_validator_config( - launcher.el_cl_genesis_data, - image, - validator_service_name, - log_level, - beacon_http_url, - el_client_context, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_validator_params, - extra_validator_labels, - persistent, - tolerations, - node_selectors, - ) - - validator_service = plan.add_service(validator_service_name, validator_config) - # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module beacon_node_identity_recipe = GetHttpRequestRecipe( endpoint="/eth/v1/node/identity", @@ -268,16 +198,6 @@ def launch( ) nodes_metrics_info = [beacon_node_metrics_info] - if validator_service: - validator_metrics_port = validator_service.ports[VALIDATOR_METRICS_PORT_ID] - validator_metrics_url = "{0}:{1}".format( - validator_service.ip_address, validator_metrics_port.number - ) - validator_node_metrics_info = node_metrics.new_node_metrics_info( - validator_service_name, METRICS_PATH, validator_metrics_url - ) - nodes_metrics_info.append(validator_node_metrics_info) - return cl_client_context.new_cl_client_context( "lighthouse", beacon_node_enr, @@ -285,7 +205,6 @@ def launch( BEACON_HTTP_PORT_NUM, nodes_metrics_info, beacon_service_name, - validator_service_name, beacon_multiaddr, beacon_peer_id, snooper_enabled, @@ -474,93 +393,6 @@ def get_beacon_config( ) -def get_validator_config( - el_cl_genesis_data, - image, - service_name, - log_level, - beacon_client_http_url, - el_client_context, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_params, - extra_labels, - persistent, - tolerations, - node_selectors, -): - validator_keys_dirpath = shared_utils.path_join( - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, - node_keystore_files.raw_keys_relative_dirpath, - ) - validator_secrets_dirpath = shared_utils.path_join( - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, - node_keystore_files.raw_secrets_relative_dirpath, - ) - - cmd = [ - "lighthouse", - "validator_client", - "--debug-level=" + log_level, - "--testnet-dir=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER, - "--validators-dir=" + validator_keys_dirpath, - # NOTE: When secrets-dir is specified, we can't add the --data-dir flag - "--secrets-dir=" + validator_secrets_dirpath, - # The node won't have a slashing protection database and will fail to start otherwise - "--init-slashing-protection", - "--http", - "--unencrypted-http-transport", - "--http-address=0.0.0.0", - "--http-port={0}".format(VALIDATOR_HTTP_PORT_NUM), - "--beacon-nodes=" + beacon_client_http_url, - # "--enable-doppelganger-protection", // Disabled to not have to wait 2 epochs before validator can start - # burn address - If unset, the validator will scream in its logs - "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, - # vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv - "--metrics", - "--metrics-address=0.0.0.0", - "--metrics-allow-origin=*", - "--metrics-port={0}".format(VALIDATOR_METRICS_PORT_NUM), - # ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^ - "--graffiti=" - + constants.CL_CLIENT_TYPE.lighthouse - + "-" - + el_client_context.client_name, - ] - - if len(extra_params): - cmd.extend([param for param in extra_params]) - - files = { - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid, - } - - return ServiceConfig( - image=image, - ports=VALIDATOR_USED_PORTS, - cmd=cmd, - files=files, - env_vars={RUST_BACKTRACE_ENVVAR_NAME: RUST_FULL_BACKTRACE_KEYWORD}, - min_cpu=v_min_cpu, - max_cpu=v_max_cpu, - min_memory=v_min_mem, - max_memory=v_max_mem, - labels=shared_utils.label_maker( - constants.CL_CLIENT_TYPE.lighthouse, - constants.CLIENT_TYPES.validator, - image, - el_client_context.client_name, - extra_labels, - ), - tolerations=tolerations, - node_selectors=node_selectors, - ) - - def new_lighthouse_launcher(el_cl_genesis_data, jwt_file, network): return struct( el_cl_genesis_data=el_cl_genesis_data, diff --git a/src/cl/lodestar/lodestar_launcher.star b/src/cl/lodestar/lodestar_launcher.star index 4ca3abd5b..dea87e67d 100644 --- a/src/cl/lodestar/lodestar_launcher.star +++ b/src/cl/lodestar/lodestar_launcher.star @@ -13,7 +13,6 @@ TCP_DISCOVERY_PORT_ID = "tcp-discovery" UDP_DISCOVERY_PORT_ID = "udp-discovery" BEACON_HTTP_PORT_ID = "http" METRICS_PORT_ID = "metrics" -VALIDATOR_METRICS_PORT_ID = "validator-metrics" # Port nums DISCOVERY_PORT_NUM = 9000 @@ -24,17 +23,6 @@ METRICS_PORT_NUM = 8008 BEACON_MIN_CPU = 50 BEACON_MIN_MEMORY = 256 -# ---------------------------------- Validator client ------------------------------------- -VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys" -VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/lodestar/validator-data" -# The min/max CPU/memory that the validator node can use -VALIDATOR_MIN_CPU = 50 -VALIDATOR_MAX_CPU = 300 -VALIDATOR_MIN_MEMORY = 128 -VALIDATOR_MAX_MEMORY = 512 - -VALIDATOR_SUFFIX_SERVICE_NAME = "validator" - METRICS_PATH = "/metrics" PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" @@ -54,13 +42,6 @@ BEACON_USED_PORTS = { ), } -VALIDATOR_USED_PORTS = { - METRICS_PORT_ID: shared_utils.new_port_spec( - METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} - - VERBOSITY_LEVELS = { constants.GLOBAL_CLIENT_LOG_LEVEL.error: "error", constants.GLOBAL_CLIENT_LOG_LEVEL.warn: "warn", @@ -84,31 +65,21 @@ def launch( bn_max_cpu, bn_min_mem, bn_max_mem, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, snooper_enabled, snooper_engine_context, blobber_enabled, blobber_extra_params, extra_beacon_params, - extra_validator_params, extra_beacon_labels, - extra_validator_labels, persistent, cl_volume_size, cl_tolerations, - validator_tolerations, participant_tolerations, global_tolerations, node_selectors, - split_mode_enabled=False, + use_separate_validator_client=True, ): beacon_service_name = "{0}".format(service_name) - validator_service_name = "{0}-{1}".format( - service_name, VALIDATOR_SUFFIX_SERVICE_NAME - ) log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS ) @@ -190,36 +161,6 @@ def launch( ) beacon_http_url = blobber_http_url - # Launch validator node if we have a keystore - if node_keystore_files != None: - v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU - v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU - v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY - v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY - tolerations = input_parser.get_client_tolerations( - validator_tolerations, participant_tolerations, global_tolerations - ) - validator_config = get_validator_config( - launcher.el_cl_genesis_data, - image, - validator_service_name, - log_level, - beacon_http_url, - el_client_context, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_validator_params, - extra_validator_labels, - persistent, - tolerations, - node_selectors, - ) - - plan.add_service(validator_service_name, validator_config) - # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module beacon_node_identity_recipe = GetHttpRequestRecipe( @@ -255,7 +196,6 @@ def launch( HTTP_PORT_NUM, nodes_metrics_info, beacon_service_name, - validator_service_name, beacon_multiaddr, beacon_peer_id, snooper_enabled, @@ -425,91 +365,6 @@ def get_beacon_config( ) -def get_validator_config( - el_cl_genesis_data, - image, - service_name, - log_level, - beacon_client_http_url, - el_client_context, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_params, - extra_labels, - persistent, - tolerations, - node_selectors, -): - root_dirpath = shared_utils.path_join( - VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER, service_name - ) - - validator_keys_dirpath = shared_utils.path_join( - VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, - node_keystore_files.raw_keys_relative_dirpath, - ) - - validator_secrets_dirpath = shared_utils.path_join( - VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, - node_keystore_files.raw_secrets_relative_dirpath, - ) - - cmd = [ - "validator", - "--logLevel=" + log_level, - # "--dataDir=" + root_dirpath, - "--paramsFile=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/config.yaml", - "--beaconNodes=" + beacon_client_http_url, - "--keystoresDir=" + validator_keys_dirpath, - "--secretsDir=" + validator_secrets_dirpath, - "--suggestedFeeRecipient=" + constants.VALIDATING_REWARDS_ACCOUNT, - # vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv - "--metrics", - "--metrics.address=0.0.0.0", - "--metrics.port={0}".format(METRICS_PORT_NUM), - # ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^ - "--graffiti=" - + constants.CL_CLIENT_TYPE.lodestar - + "-" - + el_client_context.client_name, - ] - - if len(extra_params) > 0: - # this is a repeated, we convert it into Starlark - cmd.extend([param for param in extra_params]) - - files = { - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, - } - - return ServiceConfig( - image=image, - ports=VALIDATOR_USED_PORTS, - cmd=cmd, - files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, - min_cpu=v_min_cpu, - max_cpu=v_max_cpu, - min_memory=v_min_mem, - max_memory=v_max_mem, - labels=shared_utils.label_maker( - constants.CL_CLIENT_TYPE.lodestar, - constants.CLIENT_TYPES.validator, - image, - el_client_context.client_name, - extra_labels, - ), - tolerations=tolerations, - node_selectors=node_selectors, - ) - - def new_lodestar_launcher(el_cl_genesis_data, jwt_file, network): return struct( el_cl_genesis_data=el_cl_genesis_data, diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star index 71a286c56..bd6a2b0df 100644 --- a/src/cl/nimbus/nimbus_launcher.star +++ b/src/cl/nimbus/nimbus_launcher.star @@ -29,25 +29,7 @@ DEFAULT_BEACON_IMAGE_ENTRYPOINT = ["nimbus_beacon_node"] BEACON_METRICS_PATH = "/metrics" -# ---------------------------------- Validator client ------------------------------------- VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/data/nimbus/validator-keys" -VALIDATOR_HTTP_PORT_ID = "http" -VALIDATOR_METRICS_PORT_ID = "metrics" -VALIDATOR_HTTP_PORT_NUM = 5042 -VALIDATOR_METRICS_PORT_NUM = 5064 -VALIDATOR_HTTP_PORT_WAIT_DISABLED = None - -VALIDATOR_SUFFIX_SERVICE_NAME = "validator" - -# The min/max CPU/memory that the validator node can use -VALIDATOR_MIN_CPU = 50 -VALIDATOR_MAX_CPU = 300 -VALIDATOR_MIN_MEMORY = 128 -VALIDATOR_MAX_MEMORY = 512 - -DEFAULT_VALIDATOR_IMAGE_ENTRYPOINT = ["nimbus_validator_client"] - -VALIDATOR_METRICS_PATH = "/metrics" # ---------------------------------- Genesis Files ---------------------------------- # Nimbus needs write access to the validator keys/secrets directories, and b/c the module container runs as root @@ -79,21 +61,6 @@ BEACON_USED_PORTS = { ), } - -VALIDATOR_USED_PORTS = { - VALIDATOR_HTTP_PORT_ID: shared_utils.new_port_spec( - VALIDATOR_HTTP_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.NOT_PROVIDED_APPLICATION_PROTOCOL, - VALIDATOR_HTTP_PORT_WAIT_DISABLED, - ), - VALIDATOR_METRICS_PORT_ID: shared_utils.new_port_spec( - VALIDATOR_METRICS_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), -} - VERBOSITY_LEVELS = { constants.GLOBAL_CLIENT_LOG_LEVEL.error: "ERROR", constants.GLOBAL_CLIENT_LOG_LEVEL.warn: "WARN", @@ -119,31 +86,21 @@ def launch( bn_max_cpu, bn_min_mem, bn_max_mem, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, snooper_enabled, snooper_engine_context, blobber_enabled, blobber_extra_params, extra_beacon_params, - extra_validator_params, extra_beacon_labels, - extra_validator_labels, persistent, cl_volume_size, cl_tolerations, - validator_tolerations, participant_tolerations, global_tolerations, node_selectors, - split_mode_enabled, + use_separate_validator_client, ): beacon_service_name = "{0}".format(service_name) - validator_service_name = "{0}-{1}".format( - service_name, VALIDATOR_SUFFIX_SERVICE_NAME - ) log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -193,7 +150,7 @@ def launch( snooper_engine_context, extra_beacon_params, extra_beacon_labels, - split_mode_enabled, + use_separate_validator_client, persistent, cl_volume_size, tolerations, @@ -231,47 +188,6 @@ def launch( ) nodes_metrics_info = [nimbus_node_metrics_info] - # Launch validator node if we have a keystore - validator_service = None - if node_keystore_files != None and split_mode_enabled: - v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU - v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU - v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY - v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY - tolerations = input_parser.get_client_tolerations( - validator_tolerations, participant_tolerations, global_tolerations - ) - validator_config = get_validator_config( - launcher.el_cl_genesis_data, - image, - validator_service_name, - log_level, - beacon_http_url, - el_client_context, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_validator_params, - extra_validator_labels, - persistent, - tolerations, - node_selectors, - ) - - validator_service = plan.add_service(validator_service_name, validator_config) - - if validator_service: - validator_metrics_port = validator_service.ports[VALIDATOR_METRICS_PORT_ID] - validator_metrics_url = "{0}:{1}".format( - validator_service.ip_address, validator_metrics_port.number - ) - validator_node_metrics_info = node_metrics.new_node_metrics_info( - validator_service_name, VALIDATOR_METRICS_PATH, validator_metrics_url - ) - nodes_metrics_info.append(validator_node_metrics_info) - return cl_client_context.new_cl_client_context( "nimbus", beacon_node_enr, @@ -279,7 +195,6 @@ def launch( BEACON_HTTP_PORT_NUM, nodes_metrics_info, beacon_service_name, - validator_service_name, beacon_multiaddr, beacon_peer_id, snooper_enabled, @@ -309,7 +224,7 @@ def get_beacon_config( snooper_engine_context, extra_params, extra_labels, - split_mode_enabled, + use_separate_validator_client, persistent, cl_volume_size, tolerations, @@ -383,7 +298,7 @@ def get_beacon_config( + el_client_context.client_name, ] - if node_keystore_files != None and not split_mode_enabled: + if node_keystore_files != None and not use_separate_validator_client: cmd.extend(validator_flags) if network not in constants.PUBLIC_NETWORKS: @@ -410,7 +325,7 @@ def get_beacon_config( constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file, } - if node_keystore_files != None and not split_mode_enabled: + if node_keystore_files != None and not use_separate_validator_client: files[ VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS ] = node_keystore_files.files_artifact_uuid @@ -447,81 +362,6 @@ def get_beacon_config( ) -def get_validator_config( - el_cl_genesis_data, - image, - service_name, - log_level, - beacon_http_url, - el_client_context, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_params, - extra_labels, - persistent, - tolerations, - node_selectors, -): - validator_keys_dirpath = "" - validator_secrets_dirpath = "" - if node_keystore_files != None: - validator_keys_dirpath = shared_utils.path_join( - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, - node_keystore_files.nimbus_keys_relative_dirpath, - ) - validator_secrets_dirpath = shared_utils.path_join( - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, - node_keystore_files.raw_secrets_relative_dirpath, - ) - - cmd = [ - "--beacon-node=" + beacon_http_url, - "--validators-dir=" + validator_keys_dirpath, - "--secrets-dir=" + validator_secrets_dirpath, - "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, - # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv - "--metrics", - "--metrics-address=0.0.0.0", - "--metrics-port={0}".format(VALIDATOR_METRICS_PORT_NUM), - "--graffiti=" - + constants.CL_CLIENT_TYPE.nimbus - + "-" - + el_client_context.client_name, - ] - - if len(extra_params) > 0: - cmd.extend([param for param in extra_params if param != "--split=true"]) - - files = { - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid, - } - - return ServiceConfig( - image=image, - ports=VALIDATOR_USED_PORTS, - cmd=cmd, - entrypoint=DEFAULT_VALIDATOR_IMAGE_ENTRYPOINT, - files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, - min_cpu=v_min_cpu, - max_cpu=v_max_cpu, - min_memory=v_min_mem, - max_memory=v_max_mem, - labels=shared_utils.label_maker( - constants.CL_CLIENT_TYPE.nimbus, - constants.CLIENT_TYPES.validator, - image, - el_client_context.client_name, - extra_labels, - ), - tolerations=tolerations, - node_selectors=node_selectors, - ) - - def new_nimbus_launcher(el_cl_genesis_data, jwt_file, network): return struct( el_cl_genesis_data=el_cl_genesis_data, diff --git a/src/cl/prysm/prysm_launcher.star b/src/cl/prysm/prysm_launcher.star index 697694d6f..189f6c3ab 100644 --- a/src/cl/prysm/prysm_launcher.star +++ b/src/cl/prysm/prysm_launcher.star @@ -4,8 +4,6 @@ cl_client_context = import_module("../../cl/cl_client_context.star") node_metrics = import_module("../../node_metrics_info.star") cl_node_ready_conditions = import_module("../../cl/cl_node_ready_conditions.star") constants = import_module("../../package_io/constants.star") -IMAGE_SEPARATOR_DELIMITER = "," -EXPECTED_NUM_IMAGES = 2 # ---------------------------------- Beacon client ------------------------------------- BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/prysm/beacon-data/" @@ -28,23 +26,7 @@ BEACON_MONITORING_PORT_NUM = 8080 BEACON_MIN_CPU = 100 BEACON_MIN_MEMORY = 256 -# ---------------------------------- Validator client ------------------------------------- -VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/prysm/validator-data/" -VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys" -PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/prysm-password" - -# Port IDs -VALIDATOR_MONITORING_PORT_NUM = 8081 -VALIDATOR_MONITORING_PORT_ID = "monitoring" - METRICS_PATH = "/metrics" -VALIDATOR_SUFFIX_SERVICE_NAME = "validator" - -# The min/max CPU/memory that the validator node can use -VALIDATOR_MIN_CPU = 50 -VALIDATOR_MAX_CPU = 300 -VALIDATOR_MIN_MEMORY = 64 -VALIDATOR_MAX_MEMORY = 256 MIN_PEERS = 1 @@ -67,12 +49,6 @@ BEACON_NODE_USED_PORTS = { ), } -VALIDATOR_NODE_USED_PORTS = { - VALIDATOR_MONITORING_PORT_ID: shared_utils.new_port_spec( - VALIDATOR_MONITORING_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} - VERBOSITY_LEVELS = { constants.GLOBAL_CLIENT_LOG_LEVEL.error: "error", constants.GLOBAL_CLIENT_LOG_LEVEL.warn: "warn", @@ -86,7 +62,7 @@ def launch( plan, launcher, service_name, - images, + image, participant_log_level, global_log_level, bootnode_contexts, @@ -96,46 +72,21 @@ def launch( bn_max_cpu, bn_min_mem, bn_max_mem, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, snooper_enabled, snooper_engine_context, blobber_enabled, blobber_extra_params, extra_beacon_params, - extra_validator_params, extra_beacon_labels, - extra_validator_labels, persistent, cl_volume_size, cl_tolerations, - validator_tolerations, participant_tolerations, global_tolerations, node_selectors, - split_mode_enabled=False, + use_separate_validator_client=True, ): - split_images = images.split(IMAGE_SEPARATOR_DELIMITER) - if len(split_images) != EXPECTED_NUM_IMAGES: - fail( - "Expected {0} images but got {1}".format( - EXPECTED_NUM_IMAGES, len(split_images) - ) - ) - beacon_image, validator_image = split_images - - if beacon_image.strip() == "": - fail("An empty beacon image was provided") - - if validator_image.strip() == "": - fail("An empty validator image was provided") - beacon_service_name = "{0}".format(service_name) - validator_service_name = "{0}-{1}".format( - service_name, VALIDATOR_SUFFIX_SERVICE_NAME - ) log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS ) @@ -170,7 +121,7 @@ def launch( launcher.el_cl_genesis_data, launcher.jwt_file, launcher.network, - beacon_image, + image, beacon_service_name, bootnode_contexts, el_client_context, @@ -196,40 +147,6 @@ def launch( beacon_http_endpoint = "{0}:{1}".format(beacon_service.ip_address, HTTP_PORT_NUM) beacon_rpc_endpoint = "{0}:{1}".format(beacon_service.ip_address, RPC_PORT_NUM) - # Launch validator node if we have a keystore file - validator_service = None - if node_keystore_files != None: - v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU - v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU - v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY - v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY - tolerations = input_parser.get_client_tolerations( - validator_tolerations, participant_tolerations, global_tolerations - ) - validator_config = get_validator_config( - launcher.el_cl_genesis_data, - validator_image, - validator_service_name, - log_level, - beacon_rpc_endpoint, - beacon_http_endpoint, - el_client_context, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_validator_params, - extra_validator_labels, - launcher.prysm_password_relative_filepath, - launcher.prysm_password_artifact_uuid, - persistent, - tolerations, - node_selectors, - ) - - validator_service = plan.add_service(validator_service_name, validator_config) - # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module beacon_node_identity_recipe = GetHttpRequestRecipe( endpoint="/eth/v1/node/identity", @@ -256,16 +173,6 @@ def launch( ) nodes_metrics_info = [beacon_node_metrics_info] - if validator_service: - validator_metrics_port = validator_service.ports[VALIDATOR_MONITORING_PORT_ID] - validator_metrics_url = "{0}:{1}".format( - validator_service.ip_address, validator_metrics_port.number - ) - validator_node_metrics_info = node_metrics.new_node_metrics_info( - validator_service_name, METRICS_PATH, validator_metrics_url - ) - nodes_metrics_info.append(validator_node_metrics_info) - return cl_client_context.new_cl_client_context( "prysm", beacon_node_enr, @@ -273,7 +180,6 @@ def launch( HTTP_PORT_NUM, nodes_metrics_info, beacon_service_name, - validator_service_name, beacon_multiaddr, beacon_peer_id, snooper_enabled, @@ -441,95 +347,6 @@ def get_beacon_config( ) -def get_validator_config( - el_cl_genesis_data, - validator_image, - service_name, - log_level, - beacon_rpc_endpoint, - beacon_http_endpoint, - el_client_context, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_params, - extra_labels, - prysm_password_relative_filepath, - prysm_password_artifact_uuid, - persistent, - tolerations, - node_selectors, -): - validator_keys_dirpath = shared_utils.path_join( - VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, - node_keystore_files.prysm_relative_dirpath, - ) - validator_secrets_dirpath = shared_utils.path_join( - PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, - prysm_password_relative_filepath, - ) - - cmd = [ - "--accept-terms-of-use=true", # it's mandatory in order to run the node - "--chain-config-file=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/config.yaml", - "--beacon-rpc-gateway-provider=" + beacon_http_endpoint, - "--beacon-rpc-provider=" + beacon_rpc_endpoint, - "--wallet-dir=" + validator_keys_dirpath, - "--wallet-password-file=" + validator_secrets_dirpath, - # "--datadir=" + VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER, - "--monitoring-port={0}".format(VALIDATOR_MONITORING_PORT_NUM), - "--verbosity=" + log_level, - "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, - # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv - "--disable-monitoring=false", - "--monitoring-host=0.0.0.0", - "--monitoring-port={0}".format(VALIDATOR_MONITORING_PORT_NUM), - # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^ - "--graffiti=" - + constants.CL_CLIENT_TYPE.prysm - + "-" - + el_client_context.client_name, - ] - - if len(extra_params) > 0: - # we do the for loop as otherwise its a proto repeated array - cmd.extend([param for param in extra_params]) - files = { - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, - PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: prysm_password_artifact_uuid, - } - if persistent: - files[VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory( - persistent_key="data-{0}".format(service_name) - ) - - return ServiceConfig( - image=validator_image, - ports=VALIDATOR_NODE_USED_PORTS, - cmd=cmd, - files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, - min_cpu=v_min_cpu, - max_cpu=v_max_cpu, - min_memory=v_min_mem, - max_memory=v_max_mem, - labels=shared_utils.label_maker( - constants.CL_CLIENT_TYPE.prysm, - constants.CLIENT_TYPES.validator, - validator_image, - el_client_context.client_name, - extra_labels, - ), - tolerations=tolerations, - node_selectors=node_selectors, - ) - - def new_prysm_launcher( el_cl_genesis_data, jwt_file, diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star index 80aae97b8..abc0a87f0 100644 --- a/src/cl/teku/teku_launcher.star +++ b/src/cl/teku/teku_launcher.star @@ -26,30 +26,9 @@ BEACON_MIN_CPU = 50 BEACON_MIN_MEMORY = 1024 BEACON_METRICS_PATH = "/metrics" -# ---------------------------------- Validator client ------------------------------------- -# These will get mounted as root and Teku needs directory write permissions, so we'll copy this -# into the Teku user's home directory to get around it -VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/teku/teku-validator-data" VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys" -VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys" -VALIDATOR_HTTP_PORT_ID = "http" -VALIDATOR_METRICS_PORT_ID = "metrics" -VALIDATOR_HTTP_PORT_NUM = 5042 -VALIDATOR_METRICS_PORT_NUM = 5064 -VALIDATOR_HTTP_PORT_WAIT_DISABLED = None - -VALIDATOR_SUFFIX_SERVICE_NAME = "validator" - -# The min/max CPU/memory that the validator node can use -VALIDATOR_MIN_CPU = 50 -VALIDATOR_MAX_CPU = 300 -VALIDATOR_MIN_MEMORY = 128 -VALIDATOR_MAX_MEMORY = 512 - -VALIDATOR_METRICS_PATH = "/metrics" - MIN_PEERS = 1 PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" @@ -69,20 +48,6 @@ BEACON_USED_PORTS = { ), } -VALIDATOR_USED_PORTS = { - VALIDATOR_HTTP_PORT_ID: shared_utils.new_port_spec( - VALIDATOR_HTTP_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.NOT_PROVIDED_APPLICATION_PROTOCOL, - VALIDATOR_HTTP_PORT_WAIT_DISABLED, - ), - VALIDATOR_METRICS_PORT_ID: shared_utils.new_port_spec( - VALIDATOR_METRICS_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), -} - ENTRYPOINT_ARGS = ["sh", "-c"] @@ -109,31 +74,21 @@ def launch( bn_max_cpu, bn_min_mem, bn_max_mem, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, snooper_enabled, snooper_engine_context, blobber_enabled, blobber_extra_params, extra_beacon_params, - extra_validator_params, extra_beacon_labels, - extra_validator_labels, persistent, cl_volume_size, cl_tolerations, - validator_tolerations, participant_tolerations, global_tolerations, node_selectors, - split_mode_enabled, + use_separate_validator_client, ): beacon_service_name = "{0}".format(service_name) - validator_service_name = "{0}-{1}".format( - service_name, VALIDATOR_SUFFIX_SERVICE_NAME - ) log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS ) @@ -142,9 +97,7 @@ def launch( cl_tolerations, participant_tolerations, global_tolerations ) - extra_params = [param for param in extra_beacon_params] + [ - param for param in extra_validator_params - ] + extra_params = [param for param in extra_beacon_params] network_name = shared_utils.get_network_name(launcher.network) @@ -186,7 +139,7 @@ def launch( snooper_engine_context, extra_beacon_params, extra_beacon_labels, - split_mode_enabled, + use_separate_validator_client, persistent, cl_volume_size, tolerations, @@ -226,48 +179,6 @@ def launch( ) nodes_metrics_info = [beacon_node_metrics_info] - # Launch validator node if we have a keystore - validator_service = None - if node_keystore_files != None and split_mode_enabled: - v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU - v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU - v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY - v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY - tolerations = input_parser.get_client_tolerations( - validator_tolerations, participant_tolerations, global_tolerations - ) - validator_config = get_validator_config( - launcher.el_cl_genesis_data, - image, - validator_service_name, - log_level, - beacon_http_url, - el_client_context, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - validator_service_name, - extra_validator_params, - extra_validator_labels, - persistent, - tolerations, - node_selectors, - ) - - validator_service = plan.add_service(validator_service_name, validator_config) - - if validator_service: - validator_metrics_port = validator_service.ports[VALIDATOR_METRICS_PORT_ID] - validator_metrics_url = "{0}:{1}".format( - validator_service.ip_address, validator_metrics_port.number - ) - validator_node_metrics_info = node_metrics.new_node_metrics_info( - validator_service_name, VALIDATOR_METRICS_PATH, validator_metrics_url - ) - nodes_metrics_info.append(validator_node_metrics_info) - return cl_client_context.new_cl_client_context( "teku", beacon_node_enr, @@ -275,7 +186,6 @@ def launch( BEACON_HTTP_PORT_NUM, nodes_metrics_info, beacon_service_name, - validator_service_name, multiaddr=beacon_multiaddr, peer_id=beacon_peer_id, snooper_enabled=snooper_enabled, @@ -305,7 +215,7 @@ def get_beacon_config( snooper_engine_context, extra_params, extra_labels, - split_mode_enabled, + use_separate_validator_client, persistent, cl_volume_size, tolerations, @@ -382,7 +292,7 @@ def get_beacon_config( + el_client_context.client_name, ] - if node_keystore_files != None and not split_mode_enabled: + if node_keystore_files != None and not use_separate_validator_client: cmd.extend(validator_flags) if network not in constants.PUBLIC_NETWORKS: @@ -456,7 +366,7 @@ def get_beacon_config( constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file, } - if node_keystore_files != None and not split_mode_enabled: + if node_keystore_files != None and not use_separate_validator_client: files[ VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER ] = node_keystore_files.files_artifact_uuid @@ -493,93 +403,6 @@ def get_beacon_config( ) -def get_validator_config( - el_cl_genesis_data, - image, - service_name, - log_level, - beacon_http_url, - el_client_context, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - validator_service_name, - extra_params, - extra_labels, - persistent, - tolerations, - node_selectors, -): - validator_keys_dirpath = "" - validator_secrets_dirpath = "" - if node_keystore_files != None: - validator_keys_dirpath = shared_utils.path_join( - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, - node_keystore_files.teku_keys_relative_dirpath, - ) - validator_secrets_dirpath = shared_utils.path_join( - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, - node_keystore_files.teku_secrets_relative_dirpath, - ) - - cmd = [ - "validator-client", - "--logging=" + log_level, - "--network=" - + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER - + "/config.yaml", - # "--data-path=" + VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER, - # "--data-validator-path=" + VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER, - "--beacon-node-api-endpoint=" + beacon_http_url, - "--validator-keys={0}:{1}".format( - validator_keys_dirpath, - validator_secrets_dirpath, - ), - "--validators-proposer-default-fee-recipient=" - + constants.VALIDATING_REWARDS_ACCOUNT, - "--validators-graffiti=" - + constants.CL_CLIENT_TYPE.teku - + "-" - + el_client_context.client_name, - # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv - "--metrics-enabled=true", - "--metrics-host-allowlist=*", - "--metrics-interface=0.0.0.0", - "--metrics-port={0}".format(VALIDATOR_METRICS_PORT_NUM), - ] - - if len(extra_params) > 0: - cmd.extend([param for param in extra_params if param != "--split=true"]) - - files = { - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid, - } - - return ServiceConfig( - image=image, - ports=VALIDATOR_USED_PORTS, - cmd=cmd, - files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, - min_cpu=v_min_cpu, - max_cpu=v_max_cpu, - min_memory=v_min_mem, - max_memory=v_max_mem, - labels=shared_utils.label_maker( - constants.CL_CLIENT_TYPE.teku, - constants.CLIENT_TYPES.validator, - image, - el_client_context.client_name, - extra_labels, - ), - tolerations=tolerations, - node_selectors=node_selectors, - ) - - def new_teku_launcher(el_cl_genesis_data, jwt_file, network): return struct( el_cl_genesis_data=el_cl_genesis_data, jwt_file=jwt_file, network=network diff --git a/src/package_io/constants.star b/src/package_io/constants.star index 2e9f1c06e..9d277958f 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -16,6 +16,14 @@ CL_CLIENT_TYPE = struct( lodestar="lodestar", ) +VC_CLIENT_TYPE = struct( + lighthouse="lighthouse", + lodestar="lodestar", + nimbus="nimbus", + prysm="prysm", + teku="teku", +) + GLOBAL_CLIENT_LOG_LEVEL = struct( info="info", error="error", diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 4c79f41e4..a736e3d08 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -17,10 +17,18 @@ DEFAULT_CL_IMAGES = { "lighthouse": "sigp/lighthouse:latest", "teku": "consensys/teku:latest", "nimbus": "statusim/nimbus-eth2:multiarch-latest", - "prysm": "gcr.io/prysmaticlabs/prysm/beacon-chain:latest,gcr.io/prysmaticlabs/prysm/validator:latest", + "prysm": "gcr.io/prysmaticlabs/prysm/beacon-chain:latest", "lodestar": "chainsafe/lodestar:latest", } +DEFAULT_VC_IMAGES = { + "lighthouse": "sigp/lighthouse:latest", + "lodestar": "chainsafe/lodestar:latest", + "nimbus": "statusim/nimbus-validator-client:multiarch-latest", + "prysm": "gcr.io/prysmaticlabs/prysm/validator:latest", + "teku": "consensys/teku:latest", +} + MEV_BOOST_RELAY_DEFAULT_IMAGE = "flashbots/mev-boost-relay:0.27" MEV_BOOST_RELAY_IMAGE_NON_ZERO_CAPELLA = "flashbots/mev-boost-relay:0.26" @@ -166,10 +174,13 @@ def input_parser(plan, input_args): cl_client_image=participant["cl_client_image"], cl_client_log_level=participant["cl_client_log_level"], cl_client_volume_size=participant["cl_client_volume_size"], - cl_split_mode_enabled=participant["cl_split_mode_enabled"], cl_tolerations=participant["cl_tolerations"], - tolerations=participant["tolerations"], + use_separate_validator_client=participant["use_separate_validator_client"], + validator_client_type=participant["validator_client_type"], + validator_client_image=participant["validator_client_image"], + validator_client_log_level=participant["validator_client_log_level"], validator_tolerations=participant["validator_tolerations"], + tolerations=participant["tolerations"], node_selectors=participant["node_selectors"], beacon_extra_params=participant["beacon_extra_params"], beacon_extra_labels=participant["beacon_extra_labels"], @@ -331,22 +342,13 @@ def parse_network_params(input_args): for index, participant in enumerate(result["participants"]): el_client_type = participant["el_client_type"] cl_client_type = participant["cl_client_type"] + validator_client_type = participant["validator_client_type"] if cl_client_type in (NIMBUS_NODE_NAME) and ( result["network_params"]["seconds_per_slot"] < 12 ): fail("nimbus can't be run with slot times below 12 seconds") - if participant["cl_split_mode_enabled"] and cl_client_type not in ( - "nimbus", - "teku", - ): - fail( - "split mode is only supported for nimbus and teku clients, but you specified {0}".format( - cl_client_type - ) - ) - el_image = participant["el_client_image"] if el_image == "": default_image = DEFAULT_EL_IMAGES.get(el_client_type, "") @@ -369,6 +371,30 @@ def parse_network_params(input_args): ) participant["cl_client_image"] = default_image + if participant["use_separate_validator_client"] == None: + # Default to false for CL clients that can run validator clients + # in the same process. + if cl_client_type in (constants.CL_CLIENT_TYPE.nimbus, constants.CL_CLIENT_TYPE.teku): + participant["use_separate_validator_client"] = False + else: + participant["use_separate_validator_client"] = True + + if validator_client_type == "": + # Defaults to matching the chosen CL client + validator_client_type = cl_client_type + participant["validator_client_type"] = validator_client_type + + validator_client_image = participant["validator_client_image"] + if validator_client_image == "": + default_image = DEFAULT_VC_IMAGES.get(validator_client_type, "") + if default_image == "": + fail( + "{0} received an empty image name and we don't have a default for it".format( + validator_client_type + ) + ) + participant["validator_client_image"] = default_image + snooper_enabled = participant["snooper_enabled"] if snooper_enabled == False: default_snooper_enabled = result["snooper_enabled"] @@ -591,7 +617,10 @@ def default_participant(): "cl_client_image": "", "cl_client_log_level": "", "cl_client_volume_size": 0, - "cl_split_mode_enabled": False, + "use_separate_validator_client": None, + "validator_client_type": "", + "validator_client_log_level": "", + "validator_client_image": "", "cl_tolerations": [], "validator_tolerations": [], "tolerations": [], diff --git a/src/participant.star b/src/participant.star index 315a4a51c..58ed4f4b5 100644 --- a/src/participant.star +++ b/src/participant.star @@ -1,8 +1,10 @@ def new_participant( el_client_type, cl_client_type, + validator_client_type, el_client_context, cl_client_context, + validator_client_context, snooper_engine_context, ethereum_metrics_exporter_context, xatu_sentry_context, @@ -10,8 +12,10 @@ def new_participant( return struct( el_client_type=el_client_type, cl_client_type=cl_client_type, + validator_client_type=validator_client_type, el_client_context=el_client_context, cl_client_context=cl_client_context, + validator_client_context=validator_client_context, snooper_engine_context=snooper_engine_context, ethereum_metrics_exporter_context=ethereum_metrics_exporter_context, xatu_sentry_context=xatu_sentry_context, diff --git a/src/participant_network.star b/src/participant_network.star index 13bb1e97a..c8e6e2975 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -28,6 +28,8 @@ nimbus = import_module("./cl/nimbus/nimbus_launcher.star") prysm = import_module("./cl/prysm/prysm_launcher.star") teku = import_module("./cl/teku/teku_launcher.star") +validator_client = import_module("./validator_client/validator_client_launcher.star") + snooper = import_module("./snooper/snooper_engine_launcher.star") ethereum_metrics_exporter = import_module( @@ -608,26 +610,19 @@ def launch_participant_network( participant.bn_max_cpu, participant.bn_min_mem, participant.bn_max_mem, - participant.v_min_cpu, - participant.v_max_cpu, - participant.v_min_mem, - participant.v_max_mem, participant.snooper_enabled, snooper_engine_context, participant.blobber_enabled, participant.blobber_extra_params, participant.beacon_extra_params, - participant.validator_extra_params, participant.beacon_extra_labels, - participant.validator_extra_labels, persistent, participant.cl_client_volume_size, participant.cl_tolerations, - participant.validator_tolerations, participant.tolerations, global_tolerations, node_selectors, - participant.cl_split_mode_enabled, + participant.use_separate_validator_client, ) else: boot_cl_client_ctx = all_cl_client_contexts @@ -645,26 +640,19 @@ def launch_participant_network( participant.bn_max_cpu, participant.bn_min_mem, participant.bn_max_mem, - participant.v_min_cpu, - participant.v_max_cpu, - participant.v_min_mem, - participant.v_max_mem, participant.snooper_enabled, snooper_engine_context, participant.blobber_enabled, participant.blobber_extra_params, participant.beacon_extra_params, - participant.validator_extra_params, participant.beacon_extra_labels, - participant.validator_extra_labels, persistent, participant.cl_client_volume_size, participant.cl_tolerations, - participant.validator_tolerations, participant.tolerations, global_tolerations, node_selectors, - participant.cl_split_mode_enabled, + participant.use_separate_validator_client, ) # Add participant cl additional prometheus labels @@ -725,14 +713,85 @@ def launch_participant_network( plan.print("Successfully added {0} CL participants".format(num_participants)) + all_validator_client_contexts = [] + # Some CL clients cannot run validator clients in the same process and need + # a separate validator client + _cls_that_need_separate_vc = [ + constants.CL_CLIENT_TYPE.prysm, + constants.CL_CLIENT_TYPE.lodestar, + constants.CL_CLIENT_TYPE.lighthouse + ] + for index, participant in enumerate(participants): + cl_client_type = participant.cl_client_type + validator_client_type = participant.validator_client_type + + if cl_client_type in _cls_that_need_separate_vc and not participant.use_separate_validator_client: + fail("{0} needs a separate validator client!".format(cl_client_type)) + + if not participant.use_separate_validator_client: + all_validator_client_contexts.append(None) + continue + + el_client_context = all_el_client_contexts[index] + cl_client_context = all_cl_client_contexts[index] + + # Zero-pad the index using the calculated zfill value + index_str = shared_utils.zfill_custom( + index + 1, len(str(len(participants))) + ) + + plan.print("Using separate validator client for participant #{0}".format(index_str)) + + vc_keystores = None + if participant.validator_count != 0: + vc_keystores = preregistered_validator_keys_for_nodes[ + index + ] + + validator_client_context = validator_client.launch( + plan=plan, + launcher=validator_client.new_validator_client_launcher( + el_cl_genesis_data=el_cl_data + ), + service_name="validator-client-{0}-{1}".format( + index_str, validator_client_type + ), + validator_client_type=validator_client_type, + image=participant.validator_client_image, + participant_log_level=participant.validator_client_log_level, + global_log_level=global_log_level, + cl_client_context=cl_client_context, + el_client_context=el_client_context, + node_keystore_files=vc_keystores, + v_min_cpu=participant.v_min_cpu, + v_max_cpu=participant.v_max_cpu, + v_min_mem=participant.v_min_mem, + v_max_mem=participant.v_max_mem, + extra_params=participant.validator_extra_params, + extra_labels=participant.validator_extra_labels, + prysm_password_relative_filepath=prysm_password_relative_filepath, + prysm_password_artifact_uuid=prysm_password_artifact_uuid, + validator_tolerations=participant.validator_tolerations, + participant_tolerations=participant.tolerations, + global_tolerations=global_tolerations, + node_selectors=node_selectors, + ) + all_validator_client_contexts.append(validator_client_context) + + metrics_info = validator_client_context.metrics_info + if metrics_info != None: + metrics_info["config"] = participant.prometheus_config + all_participants = [] for index, participant in enumerate(participants): el_client_type = participant.el_client_type cl_client_type = participant.cl_client_type + validator_client_type = participant.validator_client_type el_client_context = all_el_client_contexts[index] cl_client_context = all_cl_client_contexts[index] + validator_client_context = all_validator_client_contexts[index] if participant.snooper_enabled: snooper_engine_context = all_snooper_engine_contexts[index] @@ -751,8 +810,10 @@ def launch_participant_network( participant_entry = participant_module.new_participant( el_client_type, cl_client_type, + validator_client_type, el_client_context, cl_client_context, + validator_client_context, snooper_engine_context, ethereum_metrics_exporter_context, xatu_sentry_context, diff --git a/src/prometheus/prometheus_launcher.star b/src/prometheus/prometheus_launcher.star index 4e66e8505..acb83a557 100644 --- a/src/prometheus/prometheus_launcher.star +++ b/src/prometheus/prometheus_launcher.star @@ -23,6 +23,7 @@ def launch_prometheus( plan, el_client_contexts, cl_client_contexts, + validator_client_contexts, additional_metrics_jobs, ethereum_metrics_exporter_contexts, xatu_sentry_contexts, @@ -31,6 +32,7 @@ def launch_prometheus( metrics_jobs = get_metrics_jobs( el_client_contexts, cl_client_contexts, + validator_client_contexts, additional_metrics_jobs, ethereum_metrics_exporter_contexts, xatu_sentry_contexts, @@ -51,6 +53,7 @@ def launch_prometheus( def get_metrics_jobs( el_client_contexts, cl_client_contexts, + validator_client_contexts, additional_metrics_jobs, ethereum_metrics_exporter_contexts, xatu_sentry_contexts, @@ -118,38 +121,29 @@ def get_metrics_jobs( scrape_interval=scrape_interval, ) ) - if ( - len(context.cl_nodes_metrics_info) >= 2 - and context.cl_nodes_metrics_info[1] != None - ): - # Adding validator node metrics - validator_metrics_info = context.cl_nodes_metrics_info[1] - scrape_interval = PROMETHEUS_DEFAULT_SCRAPE_INTERVAL - labels = { - "service": context.validator_service_name, - "client_type": VALIDATOR_CLIENT_TYPE, - "client_name": context.client_name, - } - additional_config = validator_metrics_info[ - METRICS_INFO_ADDITIONAL_CONFIG_KEY - ] - if additional_config != None: - if additional_config.labels != None: - labels.update(additional_config.labels) - if ( - additional_config.scrape_interval != None - and additional_config.scrape_interval != "" - ): - scrape_interval = additional_config.scrape_interval - metrics_jobs.append( - new_metrics_job( - job_name=validator_metrics_info[METRICS_INFO_NAME_KEY], - endpoint=validator_metrics_info[METRICS_INFO_URL_KEY], - metrics_path=validator_metrics_info[METRICS_INFO_PATH_KEY], - labels=labels, - scrape_interval=scrape_interval, - ) + + # Adding validator clients metrics jobs + for context in validator_client_contexts: + if context == None: + continue + metrics_info = context.metrics_info + + scrape_interval = PROMETHEUS_DEFAULT_SCRAPE_INTERVAL + labels = { + "service": context.service_name, + "client_type": VALIDATOR_CLIENT_TYPE, + "client_name": context.client_name, + } + + metrics_jobs.append( + new_metrics_job( + job_name=metrics_info[METRICS_INFO_NAME_KEY], + endpoint=metrics_info[METRICS_INFO_URL_KEY], + metrics_path=metrics_info[METRICS_INFO_PATH_KEY], + labels=labels, + scrape_interval=scrape_interval, ) + ) # Adding ethereum-metrics-exporter metrics jobs for context in ethereum_metrics_exporter_contexts: diff --git a/src/validator_client/lighthouse.star b/src/validator_client/lighthouse.star new file mode 100644 index 000000000..f29583535 --- /dev/null +++ b/src/validator_client/lighthouse.star @@ -0,0 +1,101 @@ +constants = import_module("../package_io/constants.star") +input_parser = import_module("../package_io/input_parser.star") +shared_utils = import_module("../shared_utils/shared_utils.star") +validator_client_shared = import_module("./shared.star") + +RUST_BACKTRACE_ENVVAR_NAME = "RUST_BACKTRACE" +RUST_FULL_BACKTRACE_KEYWORD = "full" + +VERBOSITY_LEVELS = { + constants.GLOBAL_CLIENT_LOG_LEVEL.error: "error", + constants.GLOBAL_CLIENT_LOG_LEVEL.warn: "warn", + constants.GLOBAL_CLIENT_LOG_LEVEL.info: "info", + constants.GLOBAL_CLIENT_LOG_LEVEL.debug: "debug", + constants.GLOBAL_CLIENT_LOG_LEVEL.trace: "trace", +} + +def get_config( + el_cl_genesis_data, + image, + participant_log_level, + global_log_level, + beacon_http_url, + cl_client_context, + el_client_context, + node_keystore_files, + v_min_cpu, + v_max_cpu, + v_min_mem, + v_max_mem, + extra_params, + extra_labels, + tolerations, + node_selectors, +): + log_level = input_parser.get_client_log_level_or_default( + participant_log_level, global_log_level, VERBOSITY_LEVELS + ) + + validator_keys_dirpath = shared_utils.path_join( + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT, + node_keystore_files.raw_keys_relative_dirpath, + ) + validator_secrets_dirpath = shared_utils.path_join( + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT, + node_keystore_files.raw_secrets_relative_dirpath, + ) + + cmd = [ + "lighthouse", + "validator_client", + "--debug-level=" + log_level, + "--testnet-dir=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER, + "--validators-dir=" + validator_keys_dirpath, + # NOTE: When secrets-dir is specified, we can't add the --data-dir flag + "--secrets-dir=" + validator_secrets_dirpath, + # The node won't have a slashing protection database and will fail to start otherwise + "--init-slashing-protection", + "--beacon-nodes=" + beacon_http_url, + # "--enable-doppelganger-protection", // Disabled to not have to wait 2 epochs before validator can start + # burn address - If unset, the validator will scream in its logs + "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, + # vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv + "--metrics", + "--metrics-address=0.0.0.0", + "--metrics-allow-origin=*", + "--metrics-port={0}".format(validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + # ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^ + "--graffiti=" + + cl_client_context.client_name + + "-" + + el_client_context.client_name, + ] + + if len(extra_params): + cmd.extend([param for param in extra_params]) + + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT: node_keystore_files.files_artifact_uuid, + } + + return ServiceConfig( + image=image, + ports=validator_client_shared.VALIDATOR_CLIENT_USED_PORTS, + cmd=cmd, + files=files, + env_vars={RUST_BACKTRACE_ENVVAR_NAME: RUST_FULL_BACKTRACE_KEYWORD}, + min_cpu=v_min_cpu, + max_cpu=v_max_cpu, + min_memory=v_min_mem, + max_memory=v_max_mem, + labels=shared_utils.label_maker( + constants.VC_CLIENT_TYPE.lighthouse, + constants.CLIENT_TYPES.validator, + image, + cl_client_context.client_name, + extra_labels, + ), + tolerations=tolerations, + node_selectors=node_selectors, + ) diff --git a/src/validator_client/lodestar.star b/src/validator_client/lodestar.star new file mode 100644 index 000000000..096f37540 --- /dev/null +++ b/src/validator_client/lodestar.star @@ -0,0 +1,95 @@ +constants = import_module("../package_io/constants.star") +input_parser = import_module("../package_io/input_parser.star") +shared_utils = import_module("../shared_utils/shared_utils.star") +validator_client_shared = import_module("./shared.star") + +VERBOSITY_LEVELS = { + constants.GLOBAL_CLIENT_LOG_LEVEL.error: "error", + constants.GLOBAL_CLIENT_LOG_LEVEL.warn: "warn", + constants.GLOBAL_CLIENT_LOG_LEVEL.info: "info", + constants.GLOBAL_CLIENT_LOG_LEVEL.debug: "debug", + constants.GLOBAL_CLIENT_LOG_LEVEL.trace: "trace", +} + +def get_config( + el_cl_genesis_data, + image, + participant_log_level, + global_log_level, + beacon_http_url, + cl_client_context, + el_client_context, + node_keystore_files, + v_min_cpu, + v_max_cpu, + v_min_mem, + v_max_mem, + extra_params, + extra_labels, + tolerations, + node_selectors, +): + log_level = input_parser.get_client_log_level_or_default( + participant_log_level, global_log_level, VERBOSITY_LEVELS + ) + + validator_keys_dirpath = shared_utils.path_join( + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT, + node_keystore_files.raw_keys_relative_dirpath, + ) + + validator_secrets_dirpath = shared_utils.path_join( + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT, + node_keystore_files.raw_secrets_relative_dirpath, + ) + + cmd = [ + "validator", + "--logLevel=" + log_level, + "--paramsFile=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/config.yaml", + "--beaconNodes=" + beacon_http_url, + "--keystoresDir=" + validator_keys_dirpath, + "--secretsDir=" + validator_secrets_dirpath, + "--suggestedFeeRecipient=" + constants.VALIDATING_REWARDS_ACCOUNT, + # vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv + "--metrics", + "--metrics.address=0.0.0.0", + "--metrics.port={0}".format(validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + # ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^ + "--graffiti=" + + cl_client_context.client_name + + "-" + + el_client_context.client_name, + ] + + if len(extra_params) > 0: + # this is a repeated, we convert it into Starlark + cmd.extend([param for param in extra_params]) + + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT: node_keystore_files.files_artifact_uuid, + } + + return ServiceConfig( + image=image, + ports=validator_client_shared.VALIDATOR_CLIENT_USED_PORTS, + cmd=cmd, + files=files, + private_ip_address_placeholder=validator_client_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER, + min_cpu=v_min_cpu, + max_cpu=v_max_cpu, + min_memory=v_min_mem, + max_memory=v_max_mem, + labels=shared_utils.label_maker( + constants.VC_CLIENT_TYPE.lodestar, + constants.CLIENT_TYPES.validator, + image, + cl_client_context.client_name, + extra_labels, + ), + tolerations=tolerations, + node_selectors=node_selectors, + ) diff --git a/src/validator_client/nimbus.star b/src/validator_client/nimbus.star new file mode 100644 index 000000000..06d9638cd --- /dev/null +++ b/src/validator_client/nimbus.star @@ -0,0 +1,77 @@ +constants = import_module("../package_io/constants.star") +shared_utils = import_module("../shared_utils/shared_utils.star") +validator_client_shared = import_module("./shared.star") + + +def get_config( + el_cl_genesis_data, + image, + beacon_http_url, + cl_client_context, + el_client_context, + node_keystore_files, + v_min_cpu, + v_max_cpu, + v_min_mem, + v_max_mem, + extra_params, + extra_labels, + tolerations, + node_selectors, +): + validator_keys_dirpath = "" + validator_secrets_dirpath = "" + if node_keystore_files != None: + validator_keys_dirpath = shared_utils.path_join( + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT, + node_keystore_files.nimbus_keys_relative_dirpath, + ) + validator_secrets_dirpath = shared_utils.path_join( + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT, + node_keystore_files.raw_secrets_relative_dirpath, + ) + + cmd = [ + "--beacon-node=" + beacon_http_url, + "--validators-dir=" + validator_keys_dirpath, + "--secrets-dir=" + validator_secrets_dirpath, + "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, + # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv + "--metrics", + "--metrics-address=0.0.0.0", + "--metrics-port={0}".format(validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + "--graffiti=" + + cl_client_context.client_name + + "-" + + el_client_context.client_name, + ] + + if len(extra_params) > 0: + # this is a repeated, we convert it into Starlark + cmd.extend([param for param in extra_params]) + + files = { + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT: node_keystore_files.files_artifact_uuid, + } + + return ServiceConfig( + image=image, + ports=validator_client_shared.VALIDATOR_CLIENT_USED_PORTS, + cmd=cmd, + files=files, + private_ip_address_placeholder=validator_client_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER, + min_cpu=v_min_cpu, + max_cpu=v_max_cpu, + min_memory=v_min_mem, + max_memory=v_max_mem, + labels=shared_utils.label_maker( + constants.VC_CLIENT_TYPE.nimbus, + constants.CLIENT_TYPES.validator, + image, + cl_client_context.client_name, + extra_labels + ), + user=User(uid=0, gid=0), + tolerations=tolerations, + node_selectors=node_selectors, + ) diff --git a/src/validator_client/prysm.star b/src/validator_client/prysm.star new file mode 100644 index 000000000..e692a5468 --- /dev/null +++ b/src/validator_client/prysm.star @@ -0,0 +1,88 @@ +constants = import_module("../package_io/constants.star") +shared_utils = import_module("../shared_utils/shared_utils.star") +validator_client_shared = import_module("./shared.star") + +PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/prysm-password" +PRYSM_BEACON_RPC_PORT = 4000 + +def get_config( + el_cl_genesis_data, + image, + beacon_http_url, + cl_client_context, + el_client_context, + node_keystore_files, + v_min_cpu, + v_max_cpu, + v_min_mem, + v_max_mem, + extra_params, + extra_labels, + prysm_password_relative_filepath, + prysm_password_artifact_uuid, + tolerations, + node_selectors, +): + validator_keys_dirpath = shared_utils.path_join( + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT, + node_keystore_files.prysm_relative_dirpath, + ) + validator_secrets_dirpath = shared_utils.path_join( + PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, + prysm_password_relative_filepath, + ) + + cmd = [ + "--accept-terms-of-use=true", # it's mandatory in order to run the node + "--chain-config-file=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/config.yaml", + "--beacon-rpc-provider=" + "{}:{}".format( + cl_client_context.ip_addr, + PRYSM_BEACON_RPC_PORT, + ), + "--beacon-rest-api-provider=" + beacon_http_url, + "--wallet-dir=" + validator_keys_dirpath, + "--wallet-password-file=" + validator_secrets_dirpath, + "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, + # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv + "--disable-monitoring=false", + "--monitoring-host=0.0.0.0", + "--monitoring-port={0}".format(validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^ + "--graffiti=" + + cl_client_context.client_name + + "-" + + el_client_context.client_name, + ] + + if len(extra_params) > 0: + # this is a repeated, we convert it into Starlark + cmd.extend([param for param in extra_params]) + + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT: node_keystore_files.files_artifact_uuid, + PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: prysm_password_artifact_uuid, + } + + return ServiceConfig( + image=image, + ports=validator_client_shared.VALIDATOR_CLIENT_USED_PORTS, + cmd=cmd, + files=files, + private_ip_address_placeholder=validator_client_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER, + min_cpu=v_min_cpu, + max_cpu=v_max_cpu, + min_memory=v_min_mem, + max_memory=v_max_mem, + labels=shared_utils.label_maker( + constants.VC_CLIENT_TYPE.prysm, + constants.CLIENT_TYPES.validator, + image, + cl_client_context.client_name, + extra_labels, + ), + tolerations=tolerations, + node_selectors=node_selectors, + ) diff --git a/src/validator_client/shared.star b/src/validator_client/shared.star new file mode 100644 index 000000000..fbbf27108 --- /dev/null +++ b/src/validator_client/shared.star @@ -0,0 +1,16 @@ +shared_utils = import_module("../shared_utils/shared_utils.star") + +PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" +VALIDATOR_CLIENT_KEYS_MOUNTPOINT = "/keystores" + +VALIDATOR_CLIENT_METRICS_PORT_NUM = 8080 +VALIDATOR_CLIENT_METRICS_PORT_ID = "metrics" +METRICS_PATH = "/metrics" + +VALIDATOR_CLIENT_USED_PORTS = { + VALIDATOR_CLIENT_METRICS_PORT_ID: shared_utils.new_port_spec( + VALIDATOR_CLIENT_METRICS_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), +} diff --git a/src/validator_client/teku.star b/src/validator_client/teku.star new file mode 100644 index 000000000..d1d73e35f --- /dev/null +++ b/src/validator_client/teku.star @@ -0,0 +1,84 @@ +constants = import_module("../package_io/constants.star") +shared_utils = import_module("../shared_utils/shared_utils.star") +validator_client_shared = import_module("./shared.star") + +def get_config( + el_cl_genesis_data, + image, + beacon_http_url, + cl_client_context, + el_client_context, + node_keystore_files, + v_min_cpu, + v_max_cpu, + v_min_mem, + v_max_mem, + extra_params, + extra_labels, + tolerations, + node_selectors, +): + validator_keys_dirpath = "" + validator_secrets_dirpath = "" + if node_keystore_files != None: + validator_keys_dirpath = shared_utils.path_join( + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT, + node_keystore_files.teku_keys_relative_dirpath, + ) + validator_secrets_dirpath = shared_utils.path_join( + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT, + node_keystore_files.teku_secrets_relative_dirpath, + ) + + cmd = [ + "validator-client", + "--network=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/config.yaml", + "--beacon-node-api-endpoint=" + beacon_http_url, + "--validator-keys={0}:{1}".format( + validator_keys_dirpath, + validator_secrets_dirpath, + ), + "--validators-proposer-default-fee-recipient=" + + constants.VALIDATING_REWARDS_ACCOUNT, + "--validators-graffiti=" + + cl_client_context.client_name + + "-" + + el_client_context.client_name, + # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv + "--metrics-enabled=true", + "--metrics-host-allowlist=*", + "--metrics-interface=0.0.0.0", + "--metrics-port={0}".format(validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + ] + + if len(extra_params) > 0: + # this is a repeated, we convert it into Starlark + cmd.extend([param for param in extra_params]) + + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT: node_keystore_files.files_artifact_uuid, + } + + return ServiceConfig( + image=image, + ports=validator_client_shared.VALIDATOR_CLIENT_USED_PORTS, + cmd=cmd, + files=files, + private_ip_address_placeholder=validator_client_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER, + min_cpu=v_min_cpu, + max_cpu=v_max_cpu, + min_memory=v_min_mem, + max_memory=v_max_mem, + labels=shared_utils.label_maker( + constants.VC_CLIENT_TYPE.teku, + constants.CLIENT_TYPES.validator, + image, + cl_client_context.client_name, + extra_labels, + ), + tolerations=tolerations, + node_selectors=node_selectors, + ) diff --git a/src/validator_client/validator_client_context.star b/src/validator_client/validator_client_context.star new file mode 100644 index 000000000..07939582c --- /dev/null +++ b/src/validator_client/validator_client_context.star @@ -0,0 +1,10 @@ +def new_validator_client_context( + service_name, + client_name, + metrics_info, +): + return struct( + service_name=service_name, + client_name=client_name, + metrics_info=metrics_info, + ) diff --git a/src/validator_client/validator_client_launcher.star b/src/validator_client/validator_client_launcher.star new file mode 100644 index 000000000..58950e0d6 --- /dev/null +++ b/src/validator_client/validator_client_launcher.star @@ -0,0 +1,176 @@ +input_parser = import_module("../package_io/input_parser.star") +constants = import_module("../package_io/constants.star") +node_metrics = import_module("../node_metrics_info.star") +validator_client_context = import_module("./validator_client_context.star") + +lighthouse = import_module("./lighthouse.star") +lodestar = import_module("./lodestar.star") +nimbus = import_module("./nimbus.star") +prysm = import_module("./prysm.star") +teku = import_module("./teku.star") +validator_client_shared = import_module("./shared.star") + +# The defaults for min/max CPU/memory that the validator client can use +MIN_CPU = 50 +MAX_CPU = 300 +MIN_MEMORY = 128 +MAX_MEMORY = 512 + + +def launch( + plan, + launcher, + service_name, + validator_client_type, + image, + participant_log_level, + global_log_level, + cl_client_context, + el_client_context, + node_keystore_files, + v_min_cpu, + v_max_cpu, + v_min_mem, + v_max_mem, + extra_params, + extra_labels, + prysm_password_relative_filepath, + prysm_password_artifact_uuid, + validator_tolerations, + participant_tolerations, + global_tolerations, + node_selectors, +): + tolerations = input_parser.get_client_tolerations( + validator_tolerations, participant_tolerations, global_tolerations + ) + + beacon_http_url = "http://{}:{}".format( + cl_client_context.ip_addr, + cl_client_context.http_port_num, + ) + + v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else MIN_CPU + v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else MAX_CPU + v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else MIN_MEMORY + v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else MAX_MEMORY + + if validator_client_type == constants.VC_CLIENT_TYPE.lighthouse: + config = lighthouse.get_config( + el_cl_genesis_data=launcher.el_cl_genesis_data, + image=image, + participant_log_level=participant_log_level, + global_log_level=global_log_level, + beacon_http_url=beacon_http_url, + cl_client_context=cl_client_context, + el_client_context=el_client_context, + node_keystore_files=node_keystore_files, + v_min_cpu=v_min_cpu, + v_max_cpu=v_max_cpu, + v_min_mem=v_min_mem, + v_max_mem=v_max_mem, + extra_params=extra_params, + extra_labels=extra_labels, + tolerations=tolerations, + node_selectors=node_selectors, + ) + elif validator_client_type == constants.VC_CLIENT_TYPE.lodestar: + config = lodestar.get_config( + el_cl_genesis_data=launcher.el_cl_genesis_data, + image=image, + participant_log_level=participant_log_level, + global_log_level=global_log_level, + beacon_http_url=beacon_http_url, + cl_client_context=cl_client_context, + el_client_context=el_client_context, + node_keystore_files=node_keystore_files, + v_min_cpu=v_min_cpu, + v_max_cpu=v_max_cpu, + v_min_mem=v_min_mem, + v_max_mem=v_max_mem, + extra_params=extra_params, + extra_labels=extra_labels, + tolerations=tolerations, + node_selectors=node_selectors, + ) + elif validator_client_type == constants.VC_CLIENT_TYPE.teku: + config = teku.get_config( + el_cl_genesis_data=launcher.el_cl_genesis_data, + image=image, + beacon_http_url=beacon_http_url, + cl_client_context=cl_client_context, + el_client_context=el_client_context, + node_keystore_files=node_keystore_files, + v_min_cpu=v_min_cpu, + v_max_cpu=v_max_cpu, + v_min_mem=v_min_mem, + v_max_mem=v_max_mem, + extra_params=extra_params, + extra_labels=extra_labels, + tolerations=tolerations, + node_selectors=node_selectors, + ) + elif validator_client_type == constants.VC_CLIENT_TYPE.nimbus: + config = nimbus.get_config( + el_cl_genesis_data=launcher.el_cl_genesis_data, + image=image, + beacon_http_url=beacon_http_url, + cl_client_context=cl_client_context, + el_client_context=el_client_context, + node_keystore_files=node_keystore_files, + v_min_cpu=v_min_cpu, + v_max_cpu=v_max_cpu, + v_min_mem=v_min_mem, + v_max_mem=v_max_mem, + extra_params=extra_params, + extra_labels=extra_labels, + tolerations=tolerations, + node_selectors=node_selectors, + ) + elif validator_client_type == constants.VC_CLIENT_TYPE.prysm: + # Prysm VC only works with Prysm beacon node right now + if cl_client_context.client_name != constants.CL_CLIENT_TYPE.prysm: + fail("Prysm VC is only compatible with Prysm beacon node") + + config = prysm.get_config( + el_cl_genesis_data=launcher.el_cl_genesis_data, + image=image, + beacon_http_url=beacon_http_url, + cl_client_context=cl_client_context, + el_client_context=el_client_context, + node_keystore_files=node_keystore_files, + v_min_cpu=v_min_cpu, + v_max_cpu=v_max_cpu, + v_min_mem=v_min_mem, + v_max_mem=v_max_mem, + extra_params=extra_params, + extra_labels=extra_labels, + prysm_password_relative_filepath=prysm_password_relative_filepath, + prysm_password_artifact_uuid=prysm_password_artifact_uuid, + tolerations=tolerations, + node_selectors=node_selectors, + ) + else: + fail("Unsupported validator_client_type: {0}".format(validator_client_type)) + + validator_service = plan.add_service(service_name, config) + + validator_metrics_port = validator_service.ports[validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_ID] + validator_metrics_url = "{0}:{1}".format( + validator_service.ip_address, validator_metrics_port.number + ) + validator_node_metrics_info = node_metrics.new_node_metrics_info( + service_name, validator_client_shared.METRICS_PATH, validator_metrics_url + ) + + return validator_client_context.new_validator_client_context( + service_name=service_name, + client_name=validator_client_type, + metrics_info=validator_node_metrics_info, + ) + + +def new_validator_client_launcher(el_cl_genesis_data): + return struct( + el_cl_genesis_data=el_cl_genesis_data + ) From 02b7ed545a8c5f92942b6ba385978ce6116c44c3 Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Tue, 20 Feb 2024 18:24:07 +0100 Subject: [PATCH 02/11] Fix participant_network.star for the MEV participant --- src/participant_network.star | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/participant_network.star b/src/participant_network.star index c8e6e2975..764a4cb61 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -725,6 +725,12 @@ def launch_participant_network( cl_client_type = participant.cl_client_type validator_client_type = participant.validator_client_type + if participant.use_separate_validator_client == None: + # This should only be the case for the MEV participant, + # the regular participants default to False/True + all_validator_client_contexts.append(None) + continue + if cl_client_type in _cls_that_need_separate_vc and not participant.use_separate_validator_client: fail("{0} needs a separate validator client!".format(cl_client_type)) From e2a7e9a678ae5dadb7518ac6e65bb45cc4a23a1d Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Tue, 20 Feb 2024 20:16:55 +0100 Subject: [PATCH 03/11] Lint --- src/package_io/input_parser.star | 9 +++++++-- src/participant_network.star | 19 ++++++++++--------- src/validator_client/lighthouse.star | 5 ++++- src/validator_client/lodestar.star | 5 ++++- src/validator_client/nimbus.star | 6 ++++-- src/validator_client/prysm.star | 8 ++++++-- src/validator_client/teku.star | 5 ++++- .../validator_client_launcher.star | 8 ++++---- 8 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index a736e3d08..5b4a9bb5a 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -175,7 +175,9 @@ def input_parser(plan, input_args): cl_client_log_level=participant["cl_client_log_level"], cl_client_volume_size=participant["cl_client_volume_size"], cl_tolerations=participant["cl_tolerations"], - use_separate_validator_client=participant["use_separate_validator_client"], + use_separate_validator_client=participant[ + "use_separate_validator_client" + ], validator_client_type=participant["validator_client_type"], validator_client_image=participant["validator_client_image"], validator_client_log_level=participant["validator_client_log_level"], @@ -374,7 +376,10 @@ def parse_network_params(input_args): if participant["use_separate_validator_client"] == None: # Default to false for CL clients that can run validator clients # in the same process. - if cl_client_type in (constants.CL_CLIENT_TYPE.nimbus, constants.CL_CLIENT_TYPE.teku): + if cl_client_type in ( + constants.CL_CLIENT_TYPE.nimbus, + constants.CL_CLIENT_TYPE.teku, + ): participant["use_separate_validator_client"] = False else: participant["use_separate_validator_client"] = True diff --git a/src/participant_network.star b/src/participant_network.star index 764a4cb61..f9eca376e 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -719,7 +719,7 @@ def launch_participant_network( _cls_that_need_separate_vc = [ constants.CL_CLIENT_TYPE.prysm, constants.CL_CLIENT_TYPE.lodestar, - constants.CL_CLIENT_TYPE.lighthouse + constants.CL_CLIENT_TYPE.lighthouse, ] for index, participant in enumerate(participants): cl_client_type = participant.cl_client_type @@ -731,7 +731,10 @@ def launch_participant_network( all_validator_client_contexts.append(None) continue - if cl_client_type in _cls_that_need_separate_vc and not participant.use_separate_validator_client: + if ( + cl_client_type in _cls_that_need_separate_vc + and not participant.use_separate_validator_client + ): fail("{0} needs a separate validator client!".format(cl_client_type)) if not participant.use_separate_validator_client: @@ -742,17 +745,15 @@ def launch_participant_network( cl_client_context = all_cl_client_contexts[index] # Zero-pad the index using the calculated zfill value - index_str = shared_utils.zfill_custom( - index + 1, len(str(len(participants))) - ) + index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants)))) - plan.print("Using separate validator client for participant #{0}".format(index_str)) + plan.print( + "Using separate validator client for participant #{0}".format(index_str) + ) vc_keystores = None if participant.validator_count != 0: - vc_keystores = preregistered_validator_keys_for_nodes[ - index - ] + vc_keystores = preregistered_validator_keys_for_nodes[index] validator_client_context = validator_client.launch( plan=plan, diff --git a/src/validator_client/lighthouse.star b/src/validator_client/lighthouse.star index f29583535..2fcc833ce 100644 --- a/src/validator_client/lighthouse.star +++ b/src/validator_client/lighthouse.star @@ -14,6 +14,7 @@ VERBOSITY_LEVELS = { constants.GLOBAL_CLIENT_LOG_LEVEL.trace: "trace", } + def get_config( el_cl_genesis_data, image, @@ -63,7 +64,9 @@ def get_config( "--metrics", "--metrics-address=0.0.0.0", "--metrics-allow-origin=*", - "--metrics-port={0}".format(validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + "--metrics-port={0}".format( + validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM + ), # ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^ "--graffiti=" + cl_client_context.client_name diff --git a/src/validator_client/lodestar.star b/src/validator_client/lodestar.star index 096f37540..c60c1b025 100644 --- a/src/validator_client/lodestar.star +++ b/src/validator_client/lodestar.star @@ -11,6 +11,7 @@ VERBOSITY_LEVELS = { constants.GLOBAL_CLIENT_LOG_LEVEL.trace: "trace", } + def get_config( el_cl_genesis_data, image, @@ -56,7 +57,9 @@ def get_config( # vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv "--metrics", "--metrics.address=0.0.0.0", - "--metrics.port={0}".format(validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + "--metrics.port={0}".format( + validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM + ), # ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^ "--graffiti=" + cl_client_context.client_name diff --git a/src/validator_client/nimbus.star b/src/validator_client/nimbus.star index 06d9638cd..7a6ecaed5 100644 --- a/src/validator_client/nimbus.star +++ b/src/validator_client/nimbus.star @@ -39,7 +39,9 @@ def get_config( # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv "--metrics", "--metrics-address=0.0.0.0", - "--metrics-port={0}".format(validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + "--metrics-port={0}".format( + validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM + ), "--graffiti=" + cl_client_context.client_name + "-" @@ -69,7 +71,7 @@ def get_config( constants.CLIENT_TYPES.validator, image, cl_client_context.client_name, - extra_labels + extra_labels, ), user=User(uid=0, gid=0), tolerations=tolerations, diff --git a/src/validator_client/prysm.star b/src/validator_client/prysm.star index e692a5468..5c05ac9c1 100644 --- a/src/validator_client/prysm.star +++ b/src/validator_client/prysm.star @@ -5,6 +5,7 @@ validator_client_shared = import_module("./shared.star") PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/prysm-password" PRYSM_BEACON_RPC_PORT = 4000 + def get_config( el_cl_genesis_data, image, @@ -37,7 +38,8 @@ def get_config( "--chain-config-file=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml", - "--beacon-rpc-provider=" + "{}:{}".format( + "--beacon-rpc-provider=" + + "{}:{}".format( cl_client_context.ip_addr, PRYSM_BEACON_RPC_PORT, ), @@ -48,7 +50,9 @@ def get_config( # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv "--disable-monitoring=false", "--monitoring-host=0.0.0.0", - "--monitoring-port={0}".format(validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + "--monitoring-port={0}".format( + validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM + ), # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^ "--graffiti=" + cl_client_context.client_name diff --git a/src/validator_client/teku.star b/src/validator_client/teku.star index d1d73e35f..f644babf3 100644 --- a/src/validator_client/teku.star +++ b/src/validator_client/teku.star @@ -2,6 +2,7 @@ constants = import_module("../package_io/constants.star") shared_utils = import_module("../shared_utils/shared_utils.star") validator_client_shared = import_module("./shared.star") + def get_config( el_cl_genesis_data, image, @@ -50,7 +51,9 @@ def get_config( "--metrics-enabled=true", "--metrics-host-allowlist=*", "--metrics-interface=0.0.0.0", - "--metrics-port={0}".format(validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + "--metrics-port={0}".format( + validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM + ), ] if len(extra_params) > 0: diff --git a/src/validator_client/validator_client_launcher.star b/src/validator_client/validator_client_launcher.star index 58950e0d6..b65ce4ffd 100644 --- a/src/validator_client/validator_client_launcher.star +++ b/src/validator_client/validator_client_launcher.star @@ -155,7 +155,9 @@ def launch( validator_service = plan.add_service(service_name, config) - validator_metrics_port = validator_service.ports[validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_ID] + validator_metrics_port = validator_service.ports[ + validator_client_shared.VALIDATOR_CLIENT_METRICS_PORT_ID + ] validator_metrics_url = "{0}:{1}".format( validator_service.ip_address, validator_metrics_port.number ) @@ -171,6 +173,4 @@ def launch( def new_validator_client_launcher(el_cl_genesis_data): - return struct( - el_cl_genesis_data=el_cl_genesis_data - ) + return struct(el_cl_genesis_data=el_cl_genesis_data) From 86fde08ef65890109e14d3fa414d21dc3f24e32a Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Mon, 26 Feb 2024 17:35:49 +0100 Subject: [PATCH 04/11] feat: enable dencun-genesis --- .github/tests/dencun-genesis.yaml | 36 +++++++++++++++++++++++++++++++ src/package_io/constants.star | 2 +- src/package_io/input_parser.star | 3 --- 3 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 .github/tests/dencun-genesis.yaml diff --git a/.github/tests/dencun-genesis.yaml b/.github/tests/dencun-genesis.yaml new file mode 100644 index 000000000..e189c584c --- /dev/null +++ b/.github/tests/dencun-genesis.yaml @@ -0,0 +1,36 @@ +participants: + - el_client_type: geth + el_client_image: ethpandaops/geth:master + cl_client_type: teku + cl_client_image: ethpandaops/teku:master + - el_client_type: nethermind + el_client_image: ethpandaops/nethermind:master + cl_client_type: prysm + cl_client_image: gcr.io/prysmaticlabs/prysm/beacon-chain:latest + - el_client_type: erigon + el_client_image: ethpandaops/erigon:devel + cl_client_type: nimbus + cl_client_image: ethpandaops/nimbus:unstable + - el_client_type: besu + el_client_image: ethpandaops/besu:main + cl_client_type: lighthouse + cl_client_image: ethpandaops/lighthouse:unstable + - el_client_type: reth + el_client_image: ethpandaops/reth:main + cl_client_type: lodestar + cl_client_image: ethpandaops/lodestar:unstable + - el_client_type: ethereumjs + el_client_image: ethpandaops/ethereumjs:master + cl_client_type: teku + cl_client_image: ethpandaops/teku:master +network_params: + deneb_fork_epoch: 0 +additional_services: + - dora + - assertoor +assertoor_params: + run_stability_check: true + run_block_proposal_check: true + run_transaction_test: true + run_blob_transaction_test: true + run_opcodes_transaction_test: true diff --git a/src/package_io/constants.star b/src/package_io/constants.star index 9d277958f..65f3baec5 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -63,7 +63,7 @@ ELECTRA_FORK_VERSION = "0x60000038" ETHEREUM_GENESIS_GENERATOR = struct( bellatrix_genesis="ethpandaops/ethereum-genesis-generator:1.3.15", # EOL - capella_genesis="ethpandaops/ethereum-genesis-generator:2.0.12", # Default + capella_genesis="bbusa/egg:latest", # Default verkle_support_genesis="ethpandaops/ethereum-genesis-generator:3.0.0-rc.19", # soon to be deneb genesis verkle_genesis="ethpandaops/ethereum-genesis-generator:4.0.0-rc.6", ) diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 5b4a9bb5a..de0f987c2 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -476,9 +476,6 @@ def parse_network_params(input_args): if result["network_params"]["seconds_per_slot"] == 0: fail("seconds_per_slot is 0 needs to be > 0 ") - if result["network_params"]["deneb_fork_epoch"] == 0: - fail("deneb_fork_epoch is 0 needs to be > 0 ") - if result["network_params"]["electra_fork_epoch"] != None: # if electra is defined, then deneb needs to be set very high result["network_params"]["deneb_fork_epoch"] = HIGH_DENEB_VALUE_FORK_VERKLE From cf1891147b9ca868bba857efd27e63047c31d0ea Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 27 Feb 2024 10:26:02 +0100 Subject: [PATCH 05/11] fix fmt --- src/validator_client/validator_client_launcher.star | 1 - 1 file changed, 1 deletion(-) diff --git a/src/validator_client/validator_client_launcher.star b/src/validator_client/validator_client_launcher.star index f0e5356fa..2d0fbcc09 100644 --- a/src/validator_client/validator_client_launcher.star +++ b/src/validator_client/validator_client_launcher.star @@ -41,7 +41,6 @@ def launch( global_tolerations, node_selectors, ): - if node_keystore_files == None: return None From a675d55e59a5457e94782b87fdad0716a764e8e7 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 27 Feb 2024 10:31:22 +0100 Subject: [PATCH 06/11] fix deneb genesis img --- src/package_io/constants.star | 3 ++- src/participant_network.star | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/package_io/constants.star b/src/package_io/constants.star index 65f3baec5..b6d2686da 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -63,7 +63,8 @@ ELECTRA_FORK_VERSION = "0x60000038" ETHEREUM_GENESIS_GENERATOR = struct( bellatrix_genesis="ethpandaops/ethereum-genesis-generator:1.3.15", # EOL - capella_genesis="bbusa/egg:latest", # Default + capella_genesis="ethpandaops/ethereum-genesis-generator:2.0.12", # Default + deneb_genesis="bbusa/egg:latest", # Soon to become default verkle_support_genesis="ethpandaops/ethereum-genesis-generator:3.0.0-rc.19", # soon to be deneb genesis verkle_genesis="ethpandaops/ethereum-genesis-generator:4.0.0-rc.6", ) diff --git a/src/participant_network.star b/src/participant_network.star index aeea46048..1a2418c54 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -244,6 +244,11 @@ def launch_participant_network( ethereum_genesis_generator_image = ( constants.ETHEREUM_GENESIS_GENERATOR.capella_genesis ) + # we are running deneb genesis - experimental, soon to become default + elif network_params.deneb_fork_epoch == 0: + ethereum_genesis_generator_image = ( + constants.ETHEREUM_GENESIS_GENERATOR.deneb_genesis + ) # we are running electra - experimental elif network_params.electra_fork_epoch != None: if network_params.electra_fork_epoch == 0: From d76611ccf755dc080b4f4006918925e1147e11fd Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 27 Feb 2024 10:41:25 +0100 Subject: [PATCH 07/11] fix deneb genesis img --- src/participant_network.star | 1 + 1 file changed, 1 insertion(+) diff --git a/src/participant_network.star b/src/participant_network.star index 1a2418c54..4ac538832 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -240,6 +240,7 @@ def launch_participant_network( elif ( network_params.capella_fork_epoch == 0 and network_params.electra_fork_epoch == None + and network_params.deneb_fork_epoch > 0 ): ethereum_genesis_generator_image = ( constants.ETHEREUM_GENESIS_GENERATOR.capella_genesis From a872bc1026d53a62babf9e66f712e5e23918605e Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 27 Feb 2024 15:15:47 +0100 Subject: [PATCH 08/11] fix image name --- src/package_io/constants.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/package_io/constants.star b/src/package_io/constants.star index 74cb31295..8c45120b2 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -65,7 +65,7 @@ ELECTRA_FORK_VERSION = "0x60000038" ETHEREUM_GENESIS_GENERATOR = struct( bellatrix_genesis="ethpandaops/ethereum-genesis-generator:1.3.15", # EOL capella_genesis="ethpandaops/ethereum-genesis-generator:2.0.12", # Default - deneb_genesis="bbusa/egg:latest", # Soon to become default + deneb_genesis="ethpandaops/ethereum-genesis-generator:default-deneb-genesis", # Soon to become default verkle_support_genesis="ethpandaops/ethereum-genesis-generator:3.0.0-rc.19", # soon to be deneb genesis verkle_genesis="ethpandaops/ethereum-genesis-generator:4.0.0-rc.6", ) From 3bd8bd298ec429b04a58b4cfb4159e3c47e7a0f4 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 27 Feb 2024 15:18:30 +0100 Subject: [PATCH 09/11] fix holesky-shadowfork --- .github/tests/holesky-shadowfork.yaml_norun | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/tests/holesky-shadowfork.yaml_norun b/.github/tests/holesky-shadowfork.yaml_norun index f26bd9c8c..ca2634a03 100644 --- a/.github/tests/holesky-shadowfork.yaml_norun +++ b/.github/tests/holesky-shadowfork.yaml_norun @@ -1,10 +1,10 @@ participants: - el_client_type: geth - el_client_image: ethereum/client-go:v1.13.11 + el_client_image: ethereum/client-go:v1.13.14 cl_client_type: teku - cl_client_image: consensys/teku:24.1.1 + cl_client_image: consensys/teku:24.2.0 network_params: - dencun_fork_epoch: 1 + dencun_fork_epoch: 0 network: holesky-shadowfork additional_services: - dora From cf69d8920628c42ec66b6d0528557ac103d9e898 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 27 Feb 2024 15:19:57 +0100 Subject: [PATCH 10/11] fix test --- .github/tests/dencun-genesis.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/tests/dencun-genesis.yaml b/.github/tests/dencun-genesis.yaml index e189c584c..1c38e6597 100644 --- a/.github/tests/dencun-genesis.yaml +++ b/.github/tests/dencun-genesis.yaml @@ -25,12 +25,4 @@ participants: cl_client_image: ethpandaops/teku:master network_params: deneb_fork_epoch: 0 -additional_services: - - dora - - assertoor -assertoor_params: - run_stability_check: true - run_block_proposal_check: true - run_transaction_test: true - run_blob_transaction_test: true - run_opcodes_transaction_test: true +additional_services: [] From cfa452f4945a7c24e1232e1946da6cb85949b2c0 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 27 Feb 2024 15:49:57 +0100 Subject: [PATCH 11/11] change verbosity to detailed for runners --- .circleci/config.yml | 4 ++-- .github/workflows/nightly.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3c0057d13..7161fef3a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,14 +100,14 @@ jobs: executor: ubuntu_vm steps: - checkout - - run: kurtosis run ${PWD} "$(cat ./.github/tests/mev.yaml)" + - run: kurtosis run ${PWD} --verbosity detailed --args-file=./.github/tests/mev.yaml mix_with_tools_k8s: resource_class: xlarge executor: ubuntu_vm steps: - checkout - - run: kurtosis run ${PWD} "$(cat ./.github/tests/mix-with-tools-mev.yaml)" + - run: kurtosis run ${PWD} --verbosity detailed --args-file=./.github/tests/mev.yaml mix_persistence_k8s: resource_class: xlarge diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 75b9106ae..ffab067da 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -36,7 +36,7 @@ jobs: - name: Run Starlark run: | if [ "${{ matrix.file_name }}" != "./.github/tests/mix-with-tools-mev.yaml" ]; then - kurtosis run ${{ github.workspace }} --args-file ${{ matrix.file_name }} + kurtosis run ${{ github.workspace }} --verbosity detailed --args-file ${{ matrix.file_name }} else echo "Skipping ./.github/tests/mix-with-tools-mev.yaml" fi