feat: add support for mixed beacon-validator sim tests

.github/workflows/test-sim.yml

@@ -15,7 +15,7 @@ on:
       debug:
         description: Runtime DEBUG value
         required: false
-        default: ''
+        default: ""
 
 env:
   GETH_DOCKER_IMAGE: ethereum/client-go:v1.11.6
@@ -68,9 +68,15 @@ jobs:
         run: yarn test:sim:backup_eth_provider
         working-directory: packages/cli
 
+      # Enable these tests after fixing the following issue
+      # https://github.com/ChainSafe/lodestar/issues/5553
+      # - name: Sim tests multi client
+      #   run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:multiclient
+      #   working-directory: packages/cli
+
       - name: Upload debug log test files for "packages/cli"
         if: ${{ always() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: debug-test-logs-cli
           path: packages/cli/test-logs

packages/cli/package.json

@@ -34,6 +34,7 @@
     "test:unit": "nyc --cache-dir .nyc_output/.cache -e .ts mocha 'test/unit/**/*.test.ts'",
     "test:e2e": "mocha --timeout 30000 'test/e2e/**/*.test.ts'",
     "test:sim:multifork": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/multi_fork.test.ts",
+    "test:sim:multiclient": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/multi_client.test.ts",
     "test:sim:endpoints": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/endpoints.test.ts",
     "test:sim:deneb": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/deneb.test.ts",
     "test:sim:backup_eth_provider": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/backup_eth_provider.test.ts",

packages/cli/test/scripts/e2e_test_env.ts

@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import path from "node:path";
-import {CLClient, ELClient} from "../utils/simulation/interfaces.js";
+import {BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
 import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
 import {getEstimatedTTD, logFilesDir} from "../utils/simulation/utils/index.js";
 import {connectAllNodes} from "../utils/simulation/utils/network.js";
@@ -35,8 +35,8 @@ const env = await SimulationEnvironment.initWithDefaults(
     },
   },
   [
-    {id: "node-1", cl: CLClient.Lodestar, el: ELClient.Geth, keysCount: 32, mining: true},
-    {id: "node-2", cl: CLClient.Lodestar, el: ELClient.Nethermind, keysCount: 32},
+    {id: "node-1", beacon: BeaconClient.Lodestar, execution: ExecutionClient.Geth, keysCount: 32, mining: true},
+    {id: "node-2", beacon: BeaconClient.Lodestar, execution: ExecutionClient.Nethermind, keysCount: 32},
   ]
 );
 

packages/cli/test/sim/backup_eth_provider.test.ts

@@ -3,7 +3,7 @@ import path from "node:path";
 import {activePreset} from "@lodestar/params";
 import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";
 import {CLIQUE_SEALING_PERIOD, SIM_TESTS_SECONDS_PER_SLOT} from "../utils/simulation/constants.js";
-import {AssertionMatch, CLClient, ELClient} from "../utils/simulation/interfaces.js";
+import {AssertionMatch, BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
 import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
 import {
   getEstimatedTimeInSecForRun,
@@ -49,7 +49,7 @@ const env = await SimulationEnvironment.initWithDefaults(
       TERMINAL_TOTAL_DIFFICULTY: ttd,
     },
   },
-  [{id: "node-1", cl: CLClient.Lodestar, el: ELClient.Geth, keysCount: 32, mining: true}]
+  [{id: "node-1", beacon: BeaconClient.Lodestar, execution: ExecutionClient.Geth, keysCount: 32, mining: true}]
 );
 
 env.tracker.register({
@@ -64,8 +64,11 @@ const node2 = await env.createNodePair({
   id: "node-2",
   // As the Lodestar running on host and the geth running in docker container
   // we have to replace the IP with the local ip to connect to the geth
-  cl: {type: CLClient.Lodestar, options: {engineUrls: [replaceIpFromUrl(env.nodes[0].el.engineRpcUrl, "127.0.0.1")]}},
-  el: ELClient.Geth,
+  beacon: {
+    type: BeaconClient.Lodestar,
+    options: {engineUrls: [replaceIpFromUrl(env.nodes[0].execution.engineRpcPublicUrl, "127.0.0.1")]},
+  },
+  execution: ExecutionClient.Geth,
   keysCount: 32,
 });
 
@@ -74,8 +77,11 @@ const node3 = await env.createNodePair({
   id: "node-3",
   // As the Lodestar running on host and the geth running in docker container
   // we have to replace the IP with the local ip to connect to the geth
-  cl: {type: CLClient.Lodestar, options: {engineUrls: [replaceIpFromUrl(env.nodes[0].el.engineRpcUrl, "127.0.0.1")]}},
-  el: ELClient.Geth,
+  beacon: {
+    type: BeaconClient.Lodestar,
+    options: {engineUrls: [replaceIpFromUrl(env.nodes[0].execution.engineRpcPublicUrl, "127.0.0.1")]},
+  },
+  execution: ExecutionClient.Geth,
   keysCount: 0,
 });
 
@@ -88,8 +94,8 @@ await connectAllNodes(env.nodes);
 await waitForSlot(env.clock.getLastSlotOfEpoch(1), env.nodes, {silent: true, env});
 
 // Stop node2, node3 EL, so the only way they produce blocks is via node1 EL
-await node2.el.job.stop();
-await node3.el.job.stop();
+await node2.execution.job.stop();
+await node3.execution.job.stop();
 
 // node2 and node3 will successfully reach TTD if they can communicate to an EL on node1
 await waitForSlot(env.clock.getLastSlotOfEpoch(bellatrixForkEpoch) + activePreset.SLOTS_PER_EPOCH / 2, env.nodes, {

packages/cli/test/sim/deneb.test.ts

@@ -5,7 +5,7 @@ import {toHex, toHexString} from "@lodestar/utils";
 import {ApiError} from "@lodestar/api";
 import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";
 import {CLIQUE_SEALING_PERIOD, SIM_TESTS_SECONDS_PER_SLOT} from "../utils/simulation/constants.js";
-import {AssertionMatch, CLClient, ELClient} from "../utils/simulation/interfaces.js";
+import {AssertionMatch, BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
 import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
 import {getEstimatedTimeInSecForRun, getEstimatedTTD, logFilesDir} from "../utils/simulation/utils/index.js";
 import {connectAllNodes, connectNewNode, waitForNodeSync, waitForSlot} from "../utils/simulation/utils/network.js";
@@ -47,8 +47,8 @@ const env = await SimulationEnvironment.initWithDefaults(
     },
   },
   [
-    {id: "node-1", cl: CLClient.Lodestar, el: ELClient.Mock, keysCount: 32},
-    {id: "node-2", cl: CLClient.Lodestar, el: ELClient.Mock, keysCount: 32, remote: true},
+    {id: "node-1", beacon: BeaconClient.Lodestar, execution: ExecutionClient.Mock, keysCount: 32},
+    {id: "node-2", beacon: BeaconClient.Lodestar, execution: ExecutionClient.Mock, keysCount: 32, remote: true},
   ]
 );
 
@@ -71,36 +71,36 @@ await waitForSlot(env.clock.getLastSlotOfEpoch(bellatrixForkEpoch) + activePrese
 
 // Range Sync
 // ========================================================
-const headForRangeSync = await env.nodes[0].cl.api.beacon.getBlockHeader("head");
+const headForRangeSync = await env.nodes[0].beacon.api.beacon.getBlockHeader("head");
 ApiError.assert(headForRangeSync);
 const rangeSync = await env.createNodePair({
   id: "range-sync-node",
-  cl: CLClient.Lodestar,
-  el: ELClient.Geth,
+  beacon: BeaconClient.Lodestar,
+  execution: ExecutionClient.Geth,
   keysCount: 0,
 });
 
 // Checkpoint sync involves Weak Subjectivity Checkpoint
 // ========================================================
-const res = await env.nodes[0].cl.api.beacon.getStateFinalityCheckpoints("head");
+const res = await env.nodes[0].beacon.api.beacon.getStateFinalityCheckpoints("head");
 ApiError.assert(res);
 const headForCheckpointSync = res.response.data.finalized;
 const checkpointSync = await env.createNodePair({
   id: "checkpoint-sync-node",
-  cl: {
-    type: CLClient.Lodestar,
+  beacon: {
+    type: BeaconClient.Lodestar,
     options: {clientOptions: {wssCheckpoint: `${toHex(headForCheckpointSync.root)}:${headForCheckpointSync.epoch}`}},
   },
-  el: ELClient.Geth,
+  execution: ExecutionClient.Geth,
   keysCount: 0,
 });
 
-await rangeSync.el.job.start();
-await rangeSync.cl.job.start();
+await rangeSync.execution.job.start();
+await rangeSync.beacon.job.start();
 await connectNewNode(rangeSync, env.nodes);
 
-await checkpointSync.el.job.start();
-await checkpointSync.cl.job.start();
+await checkpointSync.execution.job.start();
+await checkpointSync.beacon.job.start();
 await connectNewNode(checkpointSync, env.nodes);
 
 await Promise.all([
@@ -114,9 +114,9 @@ await Promise.all([
   }),
 ]);
 
-await rangeSync.cl.job.stop();
-await rangeSync.el.job.stop();
-await checkpointSync.cl.job.stop();
-await checkpointSync.el.job.stop();
+await rangeSync.beacon.job.stop();
+await rangeSync.execution.job.stop();
+await checkpointSync.beacon.job.stop();
+await checkpointSync.execution.job.stop();
 
 await env.stop();

packages/cli/test/sim/endpoints.test.ts

@@ -4,7 +4,7 @@ import {expect} from "chai";
 import {toHexString} from "@chainsafe/ssz";
 import {routes} from "@lodestar/api";
 import {ApiError} from "@lodestar/api";
-import {CLClient, ELClient} from "../utils/simulation/interfaces.js";
+import {BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
 import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
 import {getEstimatedTimeInSecForRun, logFilesDir} from "../utils/simulation/utils/index.js";
 import {waitForSlot} from "../utils/simulation/utils/network.js";
@@ -36,16 +36,16 @@ const env = await SimulationEnvironment.initWithDefaults(
   [
     {
       id: "node-1",
-      cl: {type: CLClient.Lodestar, options: {clientOptions: {"sync.isSingleNode": true}}},
-      el: ELClient.Geth,
+      beacon: {type: BeaconClient.Lodestar, options: {clientOptions: {"sync.isSingleNode": true}}},
+      execution: ExecutionClient.Geth,
       keysCount: validatorCount,
       mining: true,
     },
   ]
 );
 await env.start({runTimeoutMs});
 
-const node = env.nodes[0].cl;
+const node = env.nodes[0].beacon;
 await waitForSlot(2, env.nodes, {env, silent: true});
 
 const res = await node.api.beacon.getStateValidators("head");

packages/cli/test/sim/multi_client.test.ts

@@ -0,0 +1,80 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import path from "node:path";
+import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
+import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";
+import {CLIQUE_SEALING_PERIOD, SIM_TESTS_SECONDS_PER_SLOT} from "../utils/simulation/constants.js";
+import {AssertionMatch, BeaconClient, ExecutionClient, ValidatorClient} from "../utils/simulation/interfaces.js";
+import {getEstimatedTTD, getEstimatedTimeInSecForRun, logFilesDir} from "../utils/simulation/utils/index.js";
+import {connectAllNodes, waitForSlot} from "../utils/simulation/utils/network.js";
+
+const genesisDelaySeconds = 20 * SIM_TESTS_SECONDS_PER_SLOT;
+const altairForkEpoch = 2;
+const bellatrixForkEpoch = 4;
+const capellaForkEpoch = 6;
+// Make sure bellatrix started before TTD reach
+const additionalSlotsForTTD = 2;
+const runTillEpoch = 8;
+const syncWaitEpoch = 2;
+
+const runTimeoutMs =
+  getEstimatedTimeInSecForRun({
+    genesisDelaySeconds,
+    secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
+    runTill: runTillEpoch + syncWaitEpoch,
+    // After adding Nethermind its took longer to complete
+    graceExtraTimeFraction: 0.3,
+  }) * 1000;
+
+const ttd = getEstimatedTTD({
+  genesisDelaySeconds,
+  bellatrixForkEpoch,
+  secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
+  cliqueSealingPeriod: CLIQUE_SEALING_PERIOD,
+  additionalSlots: additionalSlotsForTTD,
+});
+
+const env = await SimulationEnvironment.initWithDefaults(
+  {
+    id: "multi-clients",
+    logsDir: path.join(logFilesDir, "multi-clients"),
+    chainConfig: {
+      ALTAIR_FORK_EPOCH: altairForkEpoch,
+      BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
+      CAPELLA_FORK_EPOCH: capellaForkEpoch,
+      GENESIS_DELAY: genesisDelaySeconds,
+      TERMINAL_TOTAL_DIFFICULTY: ttd,
+    },
+  },
+  [
+    {
+      id: "node-1",
+      execution: ExecutionClient.Geth,
+      keysCount: 32,
+      mining: true,
+      beacon: BeaconClient.Lodestar,
+      validator: ValidatorClient.Lighthouse,
+    },
+    {
+      id: "node-2",
+      execution: ExecutionClient.Geth,
+      keysCount: 32,
+      remote: true,
+      beacon: BeaconClient.Lighthouse,
+      validator: ValidatorClient.Lodestar,
+    },
+  ]
+);
+
+env.tracker.register({
+  ...nodeAssertion,
+  match: ({slot}) => {
+    return slot === 1 ? AssertionMatch.Assert | AssertionMatch.Capture | AssertionMatch.Remove : AssertionMatch.None;
+  },
+});
+
+await env.start({runTimeoutMs});
+await connectAllNodes(env.nodes);
+
+await waitForSlot(env.clock.getLastSlotOfEpoch(capellaForkEpoch + 1), env.nodes, {env, silent: true});
+
+await env.stop();