From 4272ff3e27be1c85fd5e8e606b956ea31c0ae3b9 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 28 May 2024 17:09:14 +0200 Subject: [PATCH] feat: add vc_count to increase the number of validators per participant (#633) --- README.md | 4 + network_params.yaml | 1 + src/cl/cl_launcher.star | 2 +- src/package_io/input_parser.star | 26 ++ src/participant_network.star | 278 ++++++++++-------- .../validator_keystore_generator.star | 139 +++++---- 6 files changed, 273 insertions(+), 177 deletions(-) diff --git a/README.md b/README.md index e7bf04e10..bfa820bbb 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,10 @@ participants: # - teku: consensys/teku:latest vc_image: "" + # The number of validator clients to run for this participant + # Defaults to 1 + vc_count: 1 + # The log level string that this participant's CL client should log at # If this is emptystring then the global `logLevel` parameter's value will be translated into a string appropriate for the client (e.g. if # global `logLevel` = `info` then Teku would receive `INFO`, Prysm would receive `info`, etc.) diff --git a/network_params.yaml b/network_params.yaml index 355737430..a5470f3d0 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -30,6 +30,7 @@ participants: vc_type: lighthouse vc_image: sigp/lighthouse:latest vc_log_level: "" + vc_count: 1 vc_extra_env_vars: {} vc_extra_labels: {} vc_extra_params: [] diff --git a/src/cl/cl_launcher.star b/src/cl/cl_launcher.star index 87324d3de..a4e9b6ca8 100644 --- a/src/cl/cl_launcher.star +++ b/src/cl/cl_launcher.star @@ -119,7 +119,7 @@ def launch( cl_service_name = "cl-{0}-{1}-{2}".format(index_str, cl_type, el_type) new_cl_node_validator_keystores = None - if participant.validator_count != 0: + if participant.validator_count != 0 and participant.vc_count != 0: new_cl_node_validator_keystores = preregistered_validator_keys_for_nodes[ index ] diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index c32f3f367..a127e2d49 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -214,6 +214,7 @@ def input_parser(plan, input_args): vc_type=participant["vc_type"], vc_image=participant["vc_image"], vc_log_level=participant["vc_log_level"], + vc_count=participant["vc_count"], vc_tolerations=participant["vc_tolerations"], cl_extra_params=participant["cl_extra_params"], cl_extra_labels=participant["cl_extra_labels"], @@ -537,6 +538,30 @@ def parse_network_params(plan, input_args): ) participant["vc_image"] = default_image + if result["parallel_keystore_generation"] and participant["vc_count"] != 1: + fail( + "parallel_keystore_generation is only supported for 1 validator client per participant (for now)" + ) + + # If the num validator keys per node is not divisible by vc_count of a participant, fail + if ( + participant["vc_count"] > 0 + and result["network_params"]["num_validator_keys_per_node"] + % participant["vc_count"] + != 0 + ): + fail( + "num_validator_keys_per_node: {0} is not divisible by vc_count: {1} for participant: {2}".format( + result["network_params"]["num_validator_keys_per_node"], + participant["vc_count"], + str(index + 1) + + "-" + + participant["el_type"] + + "-" + + participant["cl_type"], + ) + ) + snooper_enabled = participant["snooper_enabled"] if snooper_enabled == None: participant["snooper_enabled"] = result["snooper_enabled"] @@ -826,6 +851,7 @@ def default_participant(): "vc_type": "", "vc_image": "", "vc_log_level": "", + "vc_count": 1, "vc_extra_env_vars": {}, "vc_extra_labels": {}, "vc_extra_params": [], diff --git a/src/participant_network.star b/src/participant_network.star index 1ff6fe49a..a3ce56365 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -210,150 +210,177 @@ def launch_participant_network( cl_type = participant.cl_type vc_type = participant.vc_type index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants)))) - el_context = all_el_contexts[index] - cl_context = all_cl_contexts[index] - - node_selectors = input_parser.get_client_node_selectors( - participant.node_selectors, - global_node_selectors, - ) - if participant.ethereum_metrics_exporter_enabled: - pair_name = "{0}-{1}-{2}".format(index_str, cl_type, el_type) + for sub_index in range(participant.vc_count): + el_context = all_el_contexts[index] + cl_context = all_cl_contexts[index] - ethereum_metrics_exporter_service_name = ( - "ethereum-metrics-exporter-{0}".format(pair_name) + node_selectors = input_parser.get_client_node_selectors( + participant.node_selectors, + global_node_selectors, ) + if participant.ethereum_metrics_exporter_enabled: + pair_name = "{0}-{1}-{2}".format(index_str, cl_type, el_type) - ethereum_metrics_exporter_context = ethereum_metrics_exporter.launch( - plan, - pair_name, - ethereum_metrics_exporter_service_name, - el_context, - cl_context, - node_selectors, - ) - plan.print( - "Successfully added {0} ethereum metrics exporter participants".format( - ethereum_metrics_exporter_context + ethereum_metrics_exporter_service_name = ( + "ethereum-metrics-exporter-{0}".format(pair_name) ) - ) - - all_ethereum_metrics_exporter_contexts.append(ethereum_metrics_exporter_context) - xatu_sentry_context = None - - if participant.xatu_sentry_enabled: - pair_name = "{0}-{1}-{2}".format(index_str, cl_type, el_type) - - xatu_sentry_service_name = "xatu-sentry-{0}".format(pair_name) - - xatu_sentry_context = xatu_sentry.launch( - plan, - xatu_sentry_service_name, - cl_context, - xatu_sentry_params, - network_params, - pair_name, - node_selectors, - ) - plan.print( - "Successfully added {0} xatu sentry participants".format( - xatu_sentry_context + ethereum_metrics_exporter_context = ethereum_metrics_exporter.launch( + plan, + pair_name, + ethereum_metrics_exporter_service_name, + el_context, + cl_context, + node_selectors, + ) + plan.print( + "Successfully added {0} ethereum metrics exporter participants".format( + ethereum_metrics_exporter_context + ) ) - ) - all_xatu_sentry_contexts.append(xatu_sentry_context) + all_ethereum_metrics_exporter_contexts.append( + ethereum_metrics_exporter_context + ) - plan.print("Successfully added {0} CL participants".format(num_participants)) + xatu_sentry_context = None - plan.print("Start adding validators for participant #{0}".format(index_str)) - if participant.use_separate_vc == None: - # This should only be the case for the MEV participant, - # the regular participants default to False/True - all_vc_contexts.append(None) - all_snooper_beacon_contexts.append(None) - continue + if participant.xatu_sentry_enabled: + pair_name = "{0}-{1}-{2}".format(index_str, cl_type, el_type) - if cl_type in _cls_that_need_separate_vc and not participant.use_separate_vc: - fail("{0} needs a separate validator client!".format(cl_type)) + xatu_sentry_service_name = "xatu-sentry-{0}".format(pair_name) - if not participant.use_separate_vc: - all_vc_contexts.append(None) - all_snooper_beacon_contexts.append(None) - continue + xatu_sentry_context = xatu_sentry.launch( + plan, + xatu_sentry_service_name, + cl_context, + xatu_sentry_params, + network_params, + pair_name, + node_selectors, + ) + plan.print( + "Successfully added {0} xatu sentry participants".format( + xatu_sentry_context + ) + ) - plan.print( - "Using separate validator client for participant #{0}".format(index_str) - ) + all_xatu_sentry_contexts.append(xatu_sentry_context) - vc_keystores = None - if participant.validator_count != 0: - vc_keystores = preregistered_validator_keys_for_nodes[index] + plan.print( + "Successfully added {0} CL participants".format(num_participants) + ) - vc_context = None - snooper_beacon_context = None + plan.print("Start adding validators for participant #{0}".format(index_str)) + if participant.use_separate_vc == None: + # This should only be the case for the MEV participant, + # the regular participants default to False/True + all_vc_contexts.append(None) + all_snooper_beacon_contexts.append(None) + continue + + if ( + cl_type in _cls_that_need_separate_vc + and not participant.use_separate_vc + ): + fail("{0} needs a separate validator client!".format(cl_type)) + + if not participant.use_separate_vc: + all_vc_contexts.append(None) + all_snooper_beacon_contexts.append(None) + continue - if participant.snooper_enabled: - snooper_service_name = "snooper-beacon-{0}-{1}-{2}".format( - index_str, cl_type, vc_type - ) - snooper_beacon_context = beacon_snooper.launch( - plan, - snooper_service_name, - cl_context, - node_selectors, - ) plan.print( - "Successfully added {0} snooper participants".format( - snooper_beacon_context + "Using separate validator client for participant #{0}".format(index_str) + ) + + vc_keystores = None + if participant.validator_count != 0: + if participant.vc_count == 1: + vc_keystores = preregistered_validator_keys_for_nodes[index] + else: + vc_keystores = preregistered_validator_keys_for_nodes[ + index + sub_index + ] + + vc_context = None + snooper_beacon_context = None + + if participant.snooper_enabled: + snooper_service_name = "snooper-beacon-{0}-{1}-{2}{3}".format( + index_str, + cl_type, + vc_type, + "-" + str(sub_index) if participant.vc_count != 1 else "", + ) + snooper_beacon_context = beacon_snooper.launch( + plan, + snooper_service_name, + cl_context, + node_selectors, + ) + plan.print( + "Successfully added {0} snooper participants".format( + snooper_beacon_context + ) + ) + all_snooper_beacon_contexts.append(snooper_beacon_context) + full_name = ( + "{0}-{1}-{2}-{3}{4}".format( + index_str, + el_type, + cl_type, + vc_type, + "-" + str(sub_index) if participant.vc_count != 1 else "", + ) + if participant.cl_type != participant.vc_type + else "{0}-{1}-{2}{3}".format( + index_str, + el_type, + cl_type, + "-" + str(sub_index) if participant.vc_count != 1 else "", ) ) - all_snooper_beacon_contexts.append(snooper_beacon_context) - full_name = ( - "{0}-{1}-{2}-{3}".format(index_str, el_type, cl_type, vc_type) - if participant.cl_type != participant.vc_type - else "{0}-{1}-{2}".format(index_str, el_type, cl_type) - ) - vc_context = vc.launch( - plan=plan, - launcher=vc.new_vc_launcher(el_cl_genesis_data=el_cl_data), - keymanager_file=keymanager_file, - service_name="vc-{0}".format(full_name), - vc_type=vc_type, - image=participant.vc_image, - participant_log_level=participant.vc_log_level, - global_log_level=global_log_level, - cl_context=cl_context, - el_context=el_context, - full_name=full_name, - snooper_enabled=participant.snooper_enabled, - snooper_beacon_context=snooper_beacon_context, - node_keystore_files=vc_keystores, - vc_min_cpu=participant.vc_min_cpu, - vc_max_cpu=participant.vc_max_cpu, - vc_min_mem=participant.vc_min_mem, - vc_max_mem=participant.vc_max_mem, - extra_params=participant.vc_extra_params, - extra_env_vars=participant.vc_extra_env_vars, - extra_labels=participant.vc_extra_labels, - prysm_password_relative_filepath=prysm_password_relative_filepath, - prysm_password_artifact_uuid=prysm_password_artifact_uuid, - vc_tolerations=participant.vc_tolerations, - participant_tolerations=participant.tolerations, - global_tolerations=global_tolerations, - node_selectors=node_selectors, - keymanager_enabled=participant.keymanager_enabled, - preset=network_params.preset, - network=network_params.network, - electra_fork_epoch=network_params.electra_fork_epoch, - ) - all_vc_contexts.append(vc_context) + vc_context = vc.launch( + plan=plan, + launcher=vc.new_vc_launcher(el_cl_genesis_data=el_cl_data), + keymanager_file=keymanager_file, + service_name="vc-{0}".format(full_name), + vc_type=vc_type, + image=participant.vc_image, + participant_log_level=participant.vc_log_level, + global_log_level=global_log_level, + cl_context=cl_context, + el_context=el_context, + full_name=full_name, + snooper_enabled=participant.snooper_enabled, + snooper_beacon_context=snooper_beacon_context, + node_keystore_files=vc_keystores, + vc_min_cpu=participant.vc_min_cpu, + vc_max_cpu=participant.vc_max_cpu, + vc_min_mem=participant.vc_min_mem, + vc_max_mem=participant.vc_max_mem, + extra_params=participant.vc_extra_params, + extra_env_vars=participant.vc_extra_env_vars, + extra_labels=participant.vc_extra_labels, + prysm_password_relative_filepath=prysm_password_relative_filepath, + prysm_password_artifact_uuid=prysm_password_artifact_uuid, + vc_tolerations=participant.vc_tolerations, + participant_tolerations=participant.tolerations, + global_tolerations=global_tolerations, + node_selectors=node_selectors, + keymanager_enabled=participant.keymanager_enabled, + preset=network_params.preset, + network=network_params.network, + electra_fork_epoch=network_params.electra_fork_epoch, + ) + all_vc_contexts.append(vc_context) - if vc_context and vc_context.metrics_info: - vc_context.metrics_info["config"] = participant.prometheus_config + if vc_context and vc_context.metrics_info: + vc_context.metrics_info["config"] = participant.prometheus_config - all_participants = [] + all_participants = [] for index, participant in enumerate(participants): el_type = participant.el_type @@ -364,7 +391,10 @@ def launch_participant_network( el_context = all_el_contexts[index] cl_context = all_cl_contexts[index] - vc_context = all_vc_contexts[index] + if participant.vc_count != 0: + vc_context = all_vc_contexts[index] + else: + vc_context = None if participant.snooper_enabled: snooper_engine_context = all_snooper_engine_contexts[index] diff --git a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star index 12f260875..603cfdb2d 100644 --- a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star +++ b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star @@ -2,7 +2,7 @@ shared_utils = import_module("../../shared_utils/shared_utils.star") keystore_files_module = import_module("./keystore_files.star") keystores_result = import_module("./generate_keystores_result.star") -NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR = "/node-{0}-keystores/" +NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR = "/node-{0}-keystores{1}/" # Prysm keystores are encrypted with a password PRYSM_PASSWORD = "password" @@ -85,31 +85,50 @@ def generate_validator_keystores(plan, mnemonic, participants): all_output_dirpaths = [] all_sub_command_strs = [] running_total_validator_count = 0 + for idx, participant in enumerate(participants): - output_dirpath = NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx) + output_dirpath = NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "") if participant.validator_count == 0: all_output_dirpaths.append(output_dirpath) continue - start_index = running_total_validator_count - running_total_validator_count += participant.validator_count - stop_index = start_index + participant.validator_count - generate_keystores_cmd = '{0} keystores --insecure --prysm-pass {1} --out-loc {2} --source-mnemonic "{3}" --source-min {4} --source-max {5}'.format( - KEYSTORES_GENERATION_TOOL_NAME, - PRYSM_PASSWORD, - output_dirpath, - mnemonic, - start_index, - stop_index, - ) - teku_permissions_cmd = "chmod 0777 -R " + output_dirpath + TEKU_KEYS_DIRNAME - raw_secret_permissions_cmd = ( - "chmod 0600 -R " + output_dirpath + RAW_SECRETS_DIRNAME - ) - all_sub_command_strs.append(generate_keystores_cmd) - all_sub_command_strs.append(teku_permissions_cmd) - all_sub_command_strs.append(raw_secret_permissions_cmd) - all_output_dirpaths.append(output_dirpath) + for i in range(participant.vc_count): + output_dirpath = ( + NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "-" + str(i)) + if participant.vc_count != 1 + else NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "") + ) + + start_index = running_total_validator_count + i * ( + participant.validator_count // participant.vc_count + ) + stop_index = start_index + ( + participant.validator_count // participant.vc_count + ) + + # Adjust stop_index for the last partition to include all remaining validators + if i == participant.vc_count - 1: + stop_index = running_total_validator_count + participant.validator_count + + generate_keystores_cmd = '{0} keystores --insecure --prysm-pass {1} --out-loc {2} --source-mnemonic "{3}" --source-min {4} --source-max {5}'.format( + KEYSTORES_GENERATION_TOOL_NAME, + PRYSM_PASSWORD, + output_dirpath, + mnemonic, + start_index, + stop_index, + ) + all_output_dirpaths.append(output_dirpath) + all_sub_command_strs.append(generate_keystores_cmd) + + teku_permissions_cmd = "chmod 0777 -R " + output_dirpath + TEKU_KEYS_DIRNAME + raw_secret_permissions_cmd = ( + "chmod 0600 -R " + output_dirpath + RAW_SECRETS_DIRNAME + ) + all_sub_command_strs.append(teku_permissions_cmd) + all_sub_command_strs.append(raw_secret_permissions_cmd) + + running_total_validator_count += participant.validator_count command_str = " && ".join(all_sub_command_strs) @@ -124,39 +143,57 @@ def generate_validator_keystores(plan, mnemonic, participants): keystore_files = [] running_total_validator_count = 0 for idx, participant in enumerate(participants): - output_dirpath = all_output_dirpaths[idx] if participant.validator_count == 0: keystore_files.append(None) continue - padded_idx = shared_utils.zfill_custom(idx + 1, len(str(len(participants)))) - keystore_start_index = running_total_validator_count - running_total_validator_count += participant.validator_count - keystore_stop_index = (keystore_start_index + participant.validator_count) - 1 - artifact_name = "{0}-{1}-{2}-{3}-{4}".format( - padded_idx, - participant.cl_type, - participant.el_type, - keystore_start_index, - keystore_stop_index, - ) - artifact_name = plan.store_service_files( - service_name, output_dirpath, name=artifact_name - ) - # This is necessary because the way Kurtosis currently implements artifact-storing is - base_dirname_in_artifact = shared_utils.path_base(output_dirpath) - to_add = keystore_files_module.new_keystore_files( - artifact_name, - shared_utils.path_join(base_dirname_in_artifact), - shared_utils.path_join(base_dirname_in_artifact, RAW_KEYS_DIRNAME), - shared_utils.path_join(base_dirname_in_artifact, RAW_SECRETS_DIRNAME), - shared_utils.path_join(base_dirname_in_artifact, NIMBUS_KEYS_DIRNAME), - shared_utils.path_join(base_dirname_in_artifact, PRYSM_DIRNAME), - shared_utils.path_join(base_dirname_in_artifact, TEKU_KEYS_DIRNAME), - shared_utils.path_join(base_dirname_in_artifact, TEKU_SECRETS_DIRNAME), - ) + for i in range(participant.vc_count): + output_dirpath = ( + NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "-" + str(i)) + if participant.vc_count != 1 + else NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "") + ) + padded_idx = shared_utils.zfill_custom(idx + 1, len(str(len(participants)))) + + keystore_start_index = running_total_validator_count + i * ( + participant.validator_count // participant.vc_count + ) + keystore_stop_index = keystore_start_index + ( + participant.validator_count // participant.vc_count + ) + + if i == participant.vc_count - 1: + keystore_stop_index = ( + running_total_validator_count + participant.validator_count + ) + + artifact_name = "{0}-{1}-{2}-{3}-{4}-{5}".format( + padded_idx, + participant.cl_type, + participant.el_type, + keystore_start_index, + keystore_stop_index - 1, + i, + ) + artifact_name = plan.store_service_files( + service_name, output_dirpath, name=artifact_name + ) + + base_dirname_in_artifact = shared_utils.path_base(output_dirpath) + to_add = keystore_files_module.new_keystore_files( + artifact_name, + shared_utils.path_join(base_dirname_in_artifact), + shared_utils.path_join(base_dirname_in_artifact, RAW_KEYS_DIRNAME), + shared_utils.path_join(base_dirname_in_artifact, RAW_SECRETS_DIRNAME), + shared_utils.path_join(base_dirname_in_artifact, NIMBUS_KEYS_DIRNAME), + shared_utils.path_join(base_dirname_in_artifact, PRYSM_DIRNAME), + shared_utils.path_join(base_dirname_in_artifact, TEKU_KEYS_DIRNAME), + shared_utils.path_join(base_dirname_in_artifact, TEKU_SECRETS_DIRNAME), + ) + + keystore_files.append(to_add) - keystore_files.append(to_add) + running_total_validator_count += participant.validator_count write_prysm_password_file_cmd = [ "sh", @@ -187,8 +224,6 @@ def generate_validator_keystores(plan, mnemonic, participants): keystore_files, ) - # TODO replace this with a task so that we can get the container removed - # we are removing a call to remove_service for idempotency return result @@ -204,7 +239,7 @@ def generate_valdiator_keystores_in_parallel(plan, mnemonic, participants): finished_files_to_verify = [] running_total_validator_count = 0 for idx, participant in enumerate(participants): - output_dirpath = NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx) + output_dirpath = NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "") if participant.validator_count == 0: all_generation_commands.append(None) all_output_dirpaths.append(None)