diff --git a/README.md b/README.md index 4c152c47c..2fd5138f7 100644 --- a/README.md +++ b/README.md @@ -226,10 +226,16 @@ To configure the package behaviour, you can modify your `network_params.json` fi "cl_forkmon", "el_forkmon", "beacon_metrics_gazer", - "dora", + "explorer", "prometheus_grafana" ], + // Which blockchain explorer should be used + // "dora" will use the dora explorer developped by pk910 + // "full" will use the explorer developped by the beaconcha.in team + // defaults to "light" + "explorer_version": "dora", + // If set, the package will block until a finalized epoch has occurred. "wait_for_finalization": false, diff --git a/main.star b/main.star index 2ea8259b2..86885b04c 100644 --- a/main.star +++ b/main.star @@ -31,6 +31,9 @@ beacon_metrics_gazer = import_module( dora = import_module( "github.com/kurtosis-tech/ethereum-package/src/dora/dora_launcher.star" ) +full_beaconchain_explorer = import_module( + "github.com/kurtosis-tech/ethereum-package/src/full_beaconchain/full_beaconchain_launcher.star" +) prometheus = import_module( "github.com/kurtosis-tech/ethereum-package/src/prometheus/prometheus_launcher.star" ) @@ -299,11 +302,32 @@ def run(plan, args={}): beacon_metrics_gazer_prometheus_metrics_job ) plan.print("Succesfully launched beacon metrics gazer") - elif additional_service == "dora": - plan.print("Launching dora") - dora_config_template = read_file(static_files.DORA_CONFIG_TEMPLATE_FILEPATH) - dora.launch_dora(plan, dora_config_template, all_cl_client_contexts) - plan.print("Succesfully launched dora") + elif additional_service == "explorer": + if args_with_right_defaults.explorer_version == "dora": + plan.print("Launching dora") + dora_config_template = read_file( + static_files.DORA_CONFIG_TEMPLATE_FILEPATH + ) + dora.launch_dora(plan, dora_config_template, all_cl_client_contexts) + plan.print("Succesfully launched dora") + elif args_with_right_defaults.explorer_version == "full": + plan.print("Launching full-beaconchain-explorer") + full_beaconchain_explorer_config_template = read_file( + static_files.FULL_BEACONCHAIN_CONFIG_TEMPLATE_FILEPATH + ) + full_beaconchain_explorer.launch_full_beacon( + plan, + full_beaconchain_explorer_config_template, + all_cl_client_contexts, + all_el_client_contexts, + ) + plan.print("Succesfully launched full-beaconchain-explorer") + else: + fail( + "expected explorer_version to be one of (dora, full) but got {0} which is invalid".format( + args_with_right_defaults.explorer_version + ) + ) elif additional_service == "prometheus_grafana": # Allow prometheus to be launched last so is able to collect metrics from other services launch_prometheus_grafana = True diff --git a/network_params.json b/network_params.json index 488a9b0fe..f3904591d 100644 --- a/network_params.json +++ b/network_params.json @@ -27,6 +27,16 @@ "electra_fork_epoch": null }, "launch_additional_services": true, + "additional_services": [ + "tx_spammer", + "blob_spammer", + "cl_forkmon", + "el_forkmon", + "beacon_metrics_gazer", + "explorer", + "prometheus_grafana" + ], + "explorer_version": "dora", "wait_for_finalization": false, "global_client_log_level": "info", "snooper_enabled": false, diff --git a/src/full_beaconchain/full_beaconchain_launcher.star b/src/full_beaconchain/full_beaconchain_launcher.star new file mode 100644 index 000000000..18473d8cb --- /dev/null +++ b/src/full_beaconchain/full_beaconchain_launcher.star @@ -0,0 +1,261 @@ +shared_utils = import_module( + "github.com/kurtosis-tech/ethereum-package/src/shared_utils/shared_utils.star" +) +IMAGE_NAME = "gobitfly/eth2-beaconchain-explorer:kurtosis" + +POSTGRES_PORT_ID = "postgres" +POSTGRES_PORT_NUMBER = 5432 +POSTGRES_DB = "db" +POSTGRES_USER = "postgres" +POSTGRES_PASSWORD = "pass" + +REDIS_PORT_ID = "redis" +REDIS_PORT_NUMBER = 6379 + +FRONTEND_PORT_ID = "http" +FRONTEND_PORT_NUMBER = 8080 + +LITTLE_BIGTABLE_PORT_ID = "littlebigtable" +LITTLE_BIGTABLE_PORT_NUMBER = 9000 + +FULL_BEACONCHAIN_CONFIG_FILENAME = "config.yml" + + +USED_PORTS = { + FRONTEND_PORT_ID: shared_utils.new_port_spec( + FRONTEND_PORT_NUMBER, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ) +} + + +def launch_full_beacon( + plan, + config_template, + cl_client_contexts, + el_client_contexts, +): + # TODO perhaps use the official redis & postgres packages + db_services = plan.add_services( + configs={ + # Add a Postgres server + "explorer-postgres": ServiceConfig( + image="postgres:15.2-alpine", + ports={ + POSTGRES_PORT_ID: PortSpec( + POSTGRES_PORT_NUMBER, application_protocol="postgresql" + ), + }, + env_vars={ + "POSTGRES_DB": POSTGRES_DB, + "POSTGRES_USER": POSTGRES_USER, + "POSTGRES_PASSWORD": POSTGRES_PASSWORD, + }, + ), + # Add a Redis server + "explorer-redis": ServiceConfig( + image="redis:7", + ports={ + REDIS_PORT_ID: PortSpec( + REDIS_PORT_NUMBER, application_protocol="tcp" + ), + }, + ), + # Add a Bigtable Emulator server + "explorer-littlebigtable": ServiceConfig( + image="gobitfly/little_bigtable:latest", + ports={ + LITTLE_BIGTABLE_PORT_ID: PortSpec( + LITTLE_BIGTABLE_PORT_NUMBER, application_protocol="tcp" + ), + }, + ), + } + ) + + el_uri = "http://{0}:{1}".format( + el_client_contexts[0].ip_addr, el_client_contexts[0].rpc_port_num + ) + redis_uri = "{0}:{1}".format( + db_services["explorer-redis"].ip_address, REDIS_PORT_NUMBER + ) + + template_data = new_config_template_data( + cl_client_contexts[0], + el_uri, + db_services["explorer-littlebigtable"].ip_address, + LITTLE_BIGTABLE_PORT_NUMBER, + db_services["explorer-postgres"].ip_address, + POSTGRES_PORT_NUMBER, + redis_uri, + FRONTEND_PORT_NUMBER, + ) + + template_and_data = shared_utils.new_template_and_data( + config_template, template_data + ) + template_and_data_by_rel_dest_filepath = {} + template_and_data_by_rel_dest_filepath[ + FULL_BEACONCHAIN_CONFIG_FILENAME + ] = template_and_data + + config_files_artifact_name = plan.render_templates( + template_and_data_by_rel_dest_filepath, "config.yml" + ) + + # Initialize the db schema + initdbschema = plan.add_service( + name="explorer-initdbschema", + config=ServiceConfig( + image=IMAGE_NAME, + files={ + "/app/config/": config_files_artifact_name, + }, + entrypoint=["./misc"], + cmd=["-config", "/app/config/config.yml", "-command", "applyDbSchema"], + ), + ) + # Initialize the bigtable schema + initbigtableschema = plan.add_service( + name="explorer-initbigtableschema", + config=ServiceConfig( + image=IMAGE_NAME, + files={ + "/app/config/": config_files_artifact_name, + }, + entrypoint=["./misc"], + cmd=["-config", "/app/config/config.yml", "-command", "initBigtableSchema"], + ), + ) + # Start the indexer + indexer = plan.add_service( + name="explorer-indexer", + config=ServiceConfig( + image=IMAGE_NAME, + files={ + "/app/config/": config_files_artifact_name, + }, + entrypoint=["./explorer"], + cmd=[ + "-config", + "/app/config/config.yml", + ], + env_vars={ + "INDEXER_ENABLED": "TRUE", + }, + ), + ) + # Start the eth1indexer + eth1indexer = plan.add_service( + name="explorer-eth1indexer", + config=ServiceConfig( + image=IMAGE_NAME, + files={ + "/app/config/": config_files_artifact_name, + }, + entrypoint=["./eth1indexer"], + cmd=[ + "-config", + "/app/config/config.yml", + "-blocks.concurrency", + "1", + "-blocks.tracemode", + "geth", + "-data.concurrency", + "1", + "-balances.enabled", + ], + ), + ) + + rewardsexporter = plan.add_service( + name="explorer-rewardsexporter", + config=ServiceConfig( + image=IMAGE_NAME, + files={ + "/app/config/": config_files_artifact_name, + }, + entrypoint=["./rewards-exporter"], + cmd=[ + "-config", + "/app/config/config.yml", + ], + ), + ) + + statistics = plan.add_service( + name="explorer-statistics", + config=ServiceConfig( + image=IMAGE_NAME, + files={ + "/app/config/": config_files_artifact_name, + }, + entrypoint=["./statistics"], + cmd=[ + "-config", + "/app/config/config.yml", + "-charts.enabled", + "-graffiti.enabled", + "-validators.enabled", + ], + ), + ) + + fdu = plan.add_service( + name="explorer-fdu", + config=ServiceConfig( + image=IMAGE_NAME, + files={ + "/app/config/": config_files_artifact_name, + }, + entrypoint=["./frontend-data-updater"], + cmd=[ + "-config", + "/app/config/config.yml", + ], + ), + ) + + frontend = plan.add_service( + name="explorer-frontend", + config=ServiceConfig( + image=IMAGE_NAME, + files={ + "/app/config/": config_files_artifact_name, + }, + entrypoint=["./explorer"], + cmd=[ + "-config", + "/app/config/config.yml", + ], + env_vars={ + "FRONTEND_ENABLED": "TRUE", + }, + ports={ + FRONTEND_PORT_ID: PortSpec( + FRONTEND_PORT_NUMBER, application_protocol="http" + ), + }, + ), + ) + + +def new_config_template_data( + cl_node_info, el_uri, lbt_host, lbt_port, db_host, db_port, redis_uri, frontend_port +): + return { + "CLNodeHost": cl_node_info.ip_addr, + "CLNodePort": cl_node_info.http_port_num, + "ELNodeEndpoint": el_uri, + "LBTHost": lbt_host, + "LBTPort": lbt_port, + "DBHost": db_host, + "DBPort": db_port, + "RedisEndpoint": redis_uri, + "FrontendPort": frontend_port, + } + + +def new_cl_client_info(ip_addr, port_num, service_name): + return {"IPAddr": ip_addr, "PortNum": port_num, "Name": service_name} diff --git a/src/package_io/parse_input.star b/src/package_io/parse_input.star index 425a38be7..ff0cd723d 100644 --- a/src/package_io/parse_input.star +++ b/src/package_io/parse_input.star @@ -33,7 +33,7 @@ DEFAULT_ADDITIONAL_SERVICES = [ "cl_forkmon", "el_forkmon", "beacon_metrics_gazer", - "dora", + "explorer", "prometheus_grafana", ] @@ -44,6 +44,8 @@ ATTR_TO_BE_SKIPPED_AT_ROOT = ( "tx_spammer_params", ) +DEFAULT_EXPLORER_VERSION = "dora" + package_io_constants = import_module( "github.com/kurtosis-tech/ethereum-package/src/package_io/constants.star" ) @@ -61,6 +63,7 @@ def parse_input(plan, input_args): result["mev_params"] = get_default_mev_params() result["launch_additional_services"] = True result["additional_services"] = DEFAULT_ADDITIONAL_SERVICES + result["explorer_version"] = DEFAULT_EXPLORER_VERSION for attr in input_args: value = input_args[attr] @@ -162,6 +165,7 @@ def parse_input(plan, input_args): mev_type=result["mev_type"], snooper_enabled=result["snooper_enabled"], parallel_keystore_generation=result["parallel_keystore_generation"], + explorer_version=result["explorer_version"], ) diff --git a/src/static_files/static_files.star b/src/static_files/static_files.star index 9a2252ce5..72bc200b5 100644 --- a/src/static_files/static_files.star +++ b/src/static_files/static_files.star @@ -23,6 +23,10 @@ BEACON_METRICS_GAZER_CONFIG_TEMPLATE_FILEPATH = ( DORA_CONFIG_TEMPLATE_FILEPATH = STATIC_FILES_DIRPATH + "/dora-config/config.yaml.tmpl" +FULL_BEACONCHAIN_CONFIG_TEMPLATE_FILEPATH = ( + STATIC_FILES_DIRPATH + "/full-beaconchain-config/config.yaml.tmpl" +) + # Grafana config GRAFANA_CONFIG_DIRPATH = "/grafana-config" GRAFANA_DATASOURCE_CONFIG_TEMPLATE_FILEPATH = ( diff --git a/static_files/full-beaconchain-config/config.yaml.tmpl b/static_files/full-beaconchain-config/config.yaml.tmpl new file mode 100644 index 000000000..1bd1de883 --- /dev/null +++ b/static_files/full-beaconchain-config/config.yaml.tmpl @@ -0,0 +1,62 @@ +chain: + configPath: 'node' +readerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" +writerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" +bigtable: + project: explorer + instance: explorer + emulator: true + emulatorHost: {{.LBTHost}} + emulatorPort: {{.LBTPort}} +eth1ErigonEndpoint: '{{.ELNodeEndpoint}}' +eth1GethEndpoint: '{{.ELNodeEndpoint}}' +redisCacheEndpoint: '{{.RedisEndpoint}}' +tieredCacheProvider: 'redis' +frontend: + siteDomain: "localhost:8080" + siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag + siteSubtitle: "Showing a local testnet." + server: + host: '0.0.0.0' # Address to listen on + port: '{{.FrontendPort}}' # Port to listen on + readerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" + writerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" + sessionSecret: "11111111111111111111111111111111" + jwtSigningSecret: "1111111111111111111111111111111111111111111111111111111111111111" + jwtIssuer: "localhost" + jwtValidityInMinutes: 30 + maxMailsPerEmailPerDay: 10 + mail: + mailgun: + sender: no-reply@localhost + domain: mg.localhost + privateKey: "key-11111111111111111111111111111111" + csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111' +indexer: + # fullIndexOnStartup: false # Perform a one time full db index on startup + # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup + node: + host: '{{.CLNodeHost}}' + port: '{{.CLNodePort}}' + type: lighthouse + eth1DepositContractFirstBlock: 0