fix: guid fix for besu/teku/erigon/nimbus (#443)

.github/tests/mix-persistence-k8s.yaml

@@ -1,19 +1,18 @@
 participants:
   - el_client_type: geth
-    cl_client_type: prysm
+    cl_client_type: teku
+    cl_split_mode_enabled: true
   - el_client_type: nethermind
+    cl_client_type: prysm
+  - el_client_type: erigon
     cl_client_type: nimbus
+    cl_client_image: ethpandaops/nimbus:unstable
     cl_split_mode_enabled: true
   - el_client_type: besu
     cl_client_type: lighthouse
   - el_client_type: reth
     cl_client_type: lodestar
   - el_client_type: ethereumjs
-    cl_client_type: teku
-    cl_split_mode_enabled: true
-  - el_client_type: erigon
-    cl_client_type: teku
-  - el_client_type: besu
     cl_client_type: nimbus
 additional_services: []
 persistent: true

.github/tests/mix-persistence.yaml

@@ -1,15 +1,18 @@
 participants:
   - el_client_type: geth
+    cl_client_type: teku
+    cl_split_mode_enabled: true
+  - el_client_type: nethermind
     cl_client_type: prysm
-  - el_client_type: geth
+  - el_client_type: erigon
     cl_client_type: nimbus
+    cl_client_image: ethpandaops/nimbus:unstable
     cl_split_mode_enabled: true
-  - el_client_type: nethermind
+  - el_client_type: besu
     cl_client_type: lighthouse
   - el_client_type: reth
     cl_client_type: lodestar
   - el_client_type: ethereumjs
-    cl_client_type: lighthouse
-additional_services:
-  - dora
+    cl_client_type: nimbus
+additional_services: []
 persistent: true

.github/tests/split-nimbus.yaml

@@ -1,21 +1,27 @@
 participants:
   - el_client_type: geth
     cl_client_type: nimbus
+    cl_client_image: ethpandaops/nimbus:unstable
     cl_split_mode_enabled: true
     validator_count: 0
   - el_client_type: nethermind
     cl_client_type: nimbus
     cl_split_mode_enabled: true
+    cl_client_image: ethpandaops/nimbus:unstable
   - el_client_type: erigon
     cl_client_type: nimbus
     cl_split_mode_enabled: true
+    cl_client_image: ethpandaops/nimbus:unstable
   - el_client_type: besu
     cl_client_type: nimbus
     cl_split_mode_enabled: true
+    cl_client_image: ethpandaops/nimbus:unstable
   - el_client_type: reth
     cl_client_type: nimbus
     cl_split_mode_enabled: true
+    cl_client_image: ethpandaops/nimbus:unstable
   - el_client_type: ethereumjs
     cl_client_type: nimbus
     cl_split_mode_enabled: true
+    cl_client_image: ethpandaops/nimbus:unstable
 additional_services: []

README.md

@@ -169,6 +169,7 @@ participants:
 
   # A list of optional extra params that will be passed to the CL to run separate Beacon and validator nodes
   # Only possible for nimbus or teku
+  # Please note that in order to get it to work with Nimbus, you have to use `ethpandaops/nimbus:unstable` as the image (default upstream image does not yet support this out of the box)
   # Defaults to false
   cl_split_mode_enabled: false
 

src/cl/nimbus/nimbus_launcher.star

@@ -7,6 +7,9 @@ node_metrics = import_module("../../node_metrics_info.star")
 constants = import_module("../../package_io/constants.star")
 
 #  ---------------------------------- Beacon client -------------------------------------
+# Nimbus requires that its data directory already exists (because it expects you to bind-mount it), so we
+#  have to to create it
+BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/nimbus/beacon-data"
 # Port IDs
 BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery"
 BEACON_UDP_DISCOVERY_PORT_ID = "udp-discovery"
@@ -29,7 +32,7 @@ DEFAULT_BEACON_IMAGE_ENTRYPOINT = ["nimbus_beacon_node"]
 BEACON_METRICS_PATH = "/metrics"
 
 #  ---------------------------------- Validator client -------------------------------------
-VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys"
+VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/data/nimbus/validator-keys"
 VALIDATOR_HTTP_PORT_ID = "http"
 VALIDATOR_METRICS_PORT_ID = "metrics"
 VALIDATOR_HTTP_PORT_NUM = 5042
@@ -49,13 +52,6 @@ DEFAULT_VALIDATOR_IMAGE_ENTRYPOINT = ["nimbus_validator_client"]
 VALIDATOR_METRICS_PATH = "/metrics"
 # ---------------------------------- Genesis Files ----------------------------------
 
-# Nimbus requires that its data directory already exists (because it expects you to bind-mount it), so we
-#  have to to create it
-BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/nimbus/beacon-data"
-# Nimbus wants the data dir to have these perms
-CONSENSUS_DATA_DIR_PERMS_STR = "0700"
-
-
 # Nimbus needs write access to the validator keys/secrets directories, and b/c the module container runs as root
 #  while the Nimbus container does not, we can't just point the Nimbus binary to the paths in the shared dir because
 #  it won't be able to open them. To get around this, we copy the validator keys/secrets to a path inside the Nimbus
@@ -431,6 +427,7 @@ def get_beacon_config(
             el_client_context.client_name,
             extra_labels,
         ),
+        user=User(uid=0, gid=0),
     )
 
 

src/cl/teku/teku_launcher.star

@@ -8,7 +8,7 @@ TEKU_BINARY_FILEPATH_IN_IMAGE = "/opt/teku/bin/teku"
 
 #  ---------------------------------- Beacon client -------------------------------------
 # The Docker container runs as the "teku" user so we can't write to root
-BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/opt/teku/teku-beacon-data"
+BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/teku/teku-beacon-data"
 
 # Port IDs
 BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery"
@@ -31,7 +31,7 @@ BEACON_METRICS_PATH = "/metrics"
 #  ---------------------------------- Validator client -------------------------------------
 # These will get mounted as root and Teku needs directory write permissions, so we'll copy this
 #  into the Teku user's home directory to get around it
-VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/opt/teku/teku-validator-data"
+VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/teku/teku-validator-data"
 
 VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys"
 
@@ -438,6 +438,7 @@ def get_beacon_config(
             el_client_context.client_name,
             extra_labels,
         ),
+        user=User(uid=0, gid=0),
     )
 
 

src/el/besu/besu_launcher.star

@@ -5,7 +5,7 @@ el_admin_node_info = import_module("../../el/el_admin_node_info.star")
 node_metrics = import_module("../../node_metrics_info.star")
 constants = import_module("../../package_io/constants.star")
 # The dirpath of the execution data directory on the client container
-EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/besu/execution-data"
+EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/besu/execution-data"
 
 METRICS_PATH = "/metrics"
 
@@ -254,6 +254,7 @@ def get_config(
             cl_client_name,
             extra_labels,
         ),
+        user=User(uid=0, gid=0),
     )
 
 

src/el/erigon/erigon_launcher.star

@@ -7,7 +7,7 @@ node_metrics = import_module("../../node_metrics_info.star")
 constants = import_module("../../package_io/constants.star")
 
 # The dirpath of the execution data directory on the client container
-EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/home/erigon/execution-data"
+EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/erigon/execution-data"
 
 METRICS_PATH = "/metrics"
 
@@ -193,8 +193,20 @@ def get_config(
         "--metrics",
         "--metrics.addr=0.0.0.0",
         "--metrics.port={0}".format(METRICS_PORT_NUM),
+        "--db.size.limit={0}MB".format(el_volume_size),
     ]
 
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+        constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file,
+    }
+
+    if persistent:
+        files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name),
+            size=el_volume_size,
+        )
+
     if network == "kurtosis":
         if len(existing_el_clients) > 0:
             cmd.append(
@@ -216,7 +228,6 @@ def get_config(
                 )
             )
     elif network not in constants.PUBLIC_NETWORKS:
-        cmd.append("--db.size.limit=100GB")
         cmd.append(
             "--bootnodes="
             + shared_utils.get_devnet_enodes(
@@ -240,20 +251,9 @@ def get_config(
         command_arg_str = " && ".join(command_arg)
     else:
         cmd.append("--chain={0}".format(network))
-        cmd.append("--db.size.limit=3TB")
         command_arg = cmd
         command_arg_str = " ".join(command_arg)
 
-    files = {
-        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-        constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file,
-    }
-
-    if persistent:
-        files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory(
-            persistent_key="data-{0}".format(service_name),
-            size=el_volume_size,
-        )
     return ServiceConfig(
         image=image,
         ports=USED_PORTS,
@@ -273,6 +273,7 @@ def get_config(
             cl_client_name,
             extra_labels,
         ),
+        user=User(uid=0, gid=0),
     )
 
 

src/package_io/input_parser.star

@@ -6,7 +6,7 @@ genesis_constants = import_module(
 
 DEFAULT_EL_IMAGES = {
     "geth": "ethereum/client-go:latest",
-    "erigon": "ethpandaops/erigon:2.53.0",
+    "erigon": "ethpandaops/erigon:devel",
     "nethermind": "nethermind/nethermind:latest",
     "besu": "hyperledger/besu:latest",
     "reth": "ghcr.io/paradigmxyz/reth",
@@ -16,7 +16,7 @@ DEFAULT_EL_IMAGES = {
 DEFAULT_CL_IMAGES = {
     "lighthouse": "sigp/lighthouse:latest",
     "teku": "consensys/teku:latest",
-    "nimbus": "ethpandaops/nimbus:unstable",
+    "nimbus": "statusim/nimbus-eth2:multiarch-latest",
     "prysm": "prysmaticlabs/prysm-beacon-chain:latest,prysmaticlabs/prysm-validator:latest",
     "lodestar": "chainsafe/lodestar:latest",
 }