Add support for multi client sim tests

packages/cli/package.json

@@ -34,6 +34,7 @@
     "test:unit": "nyc --cache-dir .nyc_output/.cache -e .ts mocha 'test/unit/**/*.test.ts'",
     "test:e2e": "mocha --timeout 30000 'test/e2e/**/*.test.ts'",
     "test:sim:multifork": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/multi_fork.test.ts",
+    "test:sim:multiclient": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/multi_client.test.ts",
     "test:sim:endpoints": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/endpoints.test.ts",
     "test:sim:deneb": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/deneb.test.ts",
     "test:sim:backup_eth_provider": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/backup_eth_provider.test.ts",

packages/cli/test/sim/deneb.test.ts

@@ -96,11 +96,11 @@ const checkpointSync = await env.createNodePair({
 });
 
 await rangeSync.el.job.start();
-await rangeSync.cl.job.start();
+await rangeSync.cl.beaconJob.start();
 await connectNewNode(rangeSync, env.nodes);
 
 await checkpointSync.el.job.start();
-await checkpointSync.cl.job.start();
+await checkpointSync.cl.beaconJob.start();
 await connectNewNode(checkpointSync, env.nodes);
 
 await Promise.all([
@@ -114,9 +114,9 @@ await Promise.all([
   }),
 ]);
 
-await rangeSync.cl.job.stop();
+await rangeSync.cl.beaconJob.stop();
 await rangeSync.el.job.stop();
-await checkpointSync.cl.job.stop();
+await checkpointSync.cl.beaconJob.stop();
 await checkpointSync.el.job.stop();
 
 await env.stop();

packages/cli/test/sim/multi_client.test.ts

@@ -0,0 +1,78 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import path from "node:path";
+import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
+import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";
+import {CLIQUE_SEALING_PERIOD, SIM_TESTS_SECONDS_PER_SLOT} from "../utils/simulation/constants.js";
+import {AssertionMatch, CLClient, ELClient} from "../utils/simulation/interfaces.js";
+import {getEstimatedTTD, getEstimatedTimeInSecForRun, logFilesDir} from "../utils/simulation/utils/index.js";
+import {connectAllNodes, waitForSlot} from "../utils/simulation/utils/network.js";
+
+const genesisDelaySeconds = 20 * SIM_TESTS_SECONDS_PER_SLOT;
+const altairForkEpoch = 2;
+const bellatrixForkEpoch = 4;
+const capellaForkEpoch = 6;
+// Make sure bellatrix started before TTD reach
+const additionalSlotsForTTD = 2;
+const runTillEpoch = 8;
+const syncWaitEpoch = 2;
+
+const runTimeoutMs =
+  getEstimatedTimeInSecForRun({
+    genesisDelaySeconds,
+    secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
+    runTill: runTillEpoch + syncWaitEpoch,
+    // After adding Nethermind its took longer to complete
+    graceExtraTimeFraction: 0.3,
+  }) * 1000;
+
+const ttd = getEstimatedTTD({
+  genesisDelaySeconds,
+  bellatrixForkEpoch,
+  secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
+  cliqueSealingPeriod: CLIQUE_SEALING_PERIOD,
+  additionalSlots: additionalSlotsForTTD,
+});
+
+const env = await SimulationEnvironment.initWithDefaults(
+  {
+    id: "multi-clients",
+    logsDir: path.join(logFilesDir, "multi-clients"),
+    chainConfig: {
+      ALTAIR_FORK_EPOCH: altairForkEpoch,
+      BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
+      CAPELLA_FORK_EPOCH: capellaForkEpoch,
+      GENESIS_DELAY: genesisDelaySeconds,
+      TERMINAL_TOTAL_DIFFICULTY: ttd,
+    },
+  },
+  [
+    {
+      id: "node-1",
+      el: ELClient.Geth,
+      keysCount: 32,
+      mining: true,
+      cl: {beacon: CLClient.Lodestar, validator: CLClient.Lighthouse},
+    },
+    {
+      id: "node-2",
+      el: ELClient.Geth,
+      keysCount: 32,
+      remote: true,
+      cl: {beacon: CLClient.Lighthouse, validator: CLClient.Lodestar},
+    },
+  ]
+);
+
+env.tracker.register({
+  ...nodeAssertion,
+  match: ({slot}) => {
+    return slot === 1 ? AssertionMatch.Assert | AssertionMatch.Capture | AssertionMatch.Remove : AssertionMatch.None;
+  },
+});
+
+await env.start({runTimeoutMs});
+await connectAllNodes(env.nodes);
+
+await waitForSlot(env.clock.getLastSlotOfEpoch(capellaForkEpoch + 1), env.nodes, {env, silent: true});
+
+await env.stop();

packages/cli/test/sim/multi_fork.test.ts

@@ -123,11 +123,11 @@ const checkpointSync = await env.createNodePair({
 });
 
 await rangeSync.el.job.start();
-await rangeSync.cl.job.start();
+await rangeSync.cl.beaconJob.start();
 await connectNewNode(rangeSync, env.nodes);
 
 await checkpointSync.el.job.start();
-await checkpointSync.cl.job.start();
+await checkpointSync.cl.beaconJob.start();
 await connectNewNode(checkpointSync, env.nodes);
 
 await Promise.all([
@@ -141,9 +141,9 @@ await Promise.all([
   }),
 ]);
 
-await rangeSync.cl.job.stop();
+await rangeSync.cl.beaconJob.stop();
 await rangeSync.el.job.stop();
-await checkpointSync.cl.job.stop();
+await checkpointSync.cl.beaconJob.stop();
 await checkpointSync.el.job.stop();
 
 // Unknown block sync
@@ -158,7 +158,7 @@ const unknownBlockSync = await env.createNodePair({
   keysCount: 0,
 });
 await unknownBlockSync.el.job.start();
-await unknownBlockSync.cl.job.start();
+await unknownBlockSync.cl.beaconJob.start();
 const headForUnknownBlockSync = await env.nodes[0].cl.api.beacon.getBlockV2("head");
 ApiError.assert(headForUnknownBlockSync);
 await connectNewNode(unknownBlockSync, env.nodes);

packages/cli/test/utils/simulation/SimulationEnvironment.ts

@@ -160,7 +160,8 @@ export class SimulationEnvironment {
         throw new Error("The genesis state for CL clients is not defined.");
       }
 
-      await Promise.all(this.nodes.map((node) => node.cl.job.start()));
+      await Promise.all(this.nodes.map((node) => node.cl.beaconJob?.start()));
+      await Promise.all(this.nodes.map((node) => node.cl.validatorJob?.start()));
 
       if (this.nodes.some((node) => node.cl.keys.type === "remote")) {
         console.log("Starting external signer...");
@@ -194,8 +195,9 @@ export class SimulationEnvironment {
     process.removeAllListeners("SIGINT");
     console.log(`Simulation environment "${this.options.id}" is stopping: ${message}`);
     await this.tracker.stop();
+    await Promise.all(this.nodes.map((node) => node.cl.validatorJob?.stop()));
+    await Promise.all(this.nodes.map((node) => node.cl.beaconJob?.stop()));
     await Promise.all(this.nodes.map((node) => node.el.job.stop()));
-    await Promise.all(this.nodes.map((node) => node.cl.job.stop()));
     await this.externalSigner.stop();
     await this.runner.stop();
     this.options.controller.abort();
@@ -208,14 +210,14 @@ export class SimulationEnvironment {
     }
   }
 
-  async createNodePair<C extends CLClient, E extends ELClient>({
+  async createNodePair<B extends CLClient, V extends CLClient, E extends ELClient>({
     el,
     cl,
     keysCount,
     id,
     remote,
     mining,
-  }: NodePairOptions<C, E>): Promise<NodePair> {
+  }: NodePairOptions<B, V, E>): Promise<NodePair> {
     if (this.genesisState && keysCount > 0) {
       throw new Error("Genesis state already initialized. Can not add more keys to it.");
     }
@@ -231,9 +233,8 @@ export class SimulationEnvironment {
         ? {type: "local", secretKeys: interopKeys}
         : {type: "no-keys"};
 
+    // Execution Node
     const elType = typeof el === "object" ? el.type : el;
-    const clType = typeof cl === "object" ? cl.type : cl;
-
     const elOptions = typeof el === "object" ? el.options : {};
     const elNode = await createELNode(elType, {
       ...elOptions,
@@ -252,15 +253,21 @@ export class SimulationEnvironment {
       clientOptions: elOptions.clientOptions,
     });
 
-    const clOptions = typeof cl === "object" ? cl.options : {};
+    // Beacon Node
+    const {beacon, beaconOptions, validator, validatorOptions} =
+      typeof cl === "object"
+        ? "type" in cl
+          ? {beacon: cl.type, validator: cl.type, beaconOptions: cl.options, validatorOptions: cl.options}
+          : cl
+        : {beacon: cl, validator: cl, beaconOptions: {}, validatorOptions: {}};
+
     const engineUrls = [
       // As lodestar is running on host machine, need to connect through local exposed ports
-      clType === CLClient.Lodestar ? replaceIpFromUrl(elNode.engineRpcUrl, "127.0.0.1") : elNode.engineRpcUrl,
-      ...(clOptions.engineUrls || []),
+      beacon === CLClient.Lodestar ? replaceIpFromUrl(elNode.engineRpcUrl, "127.0.0.1") : elNode.engineRpcUrl,
+      ...(beaconOptions?.engineUrls ?? []),
     ];
 
-    const clNode = await createCLNode(clType, {
-      ...clOptions,
+    const commonOptions = {
       id,
       keys,
       engineMock: typeof el === "string" ? el === ELClient.Mock : el.type === ELClient.Mock,
@@ -270,13 +277,21 @@ export class SimulationEnvironment {
       runner: this.runner,
       genesisTime: this.options.elGenesisTime,
       genesisState: this.genesisState,
-      paths: getCLNodePaths({
-        root: this.options.rootDir,
-        id,
-        client: clType,
-        logsDir: this.options.logsDir,
-      }),
-      clientOptions: clOptions.clientOptions,
+    };
+
+    const clNode = await createCLNode({
+      beacon,
+      beaconOptions: {
+        ...beaconOptions,
+        ...commonOptions,
+        paths: getCLNodePaths({id, logsDir: this.options.logsDir, client: beacon, root: this.options.rootDir}),
+      },
+      validator,
+      validatorOptions: {
+        ...validatorOptions,
+        ...commonOptions,
+        paths: getCLNodePaths({id, logsDir: this.options.logsDir, client: validator, root: this.options.rootDir}),
+      },
     });
 
     this.nodePairCount += 1;

packages/cli/test/utils/simulation/cl_clients/index.ts

@@ -8,15 +8,42 @@ import {createCLNodePaths} from "../utils/paths.js";
 import {generateLighthouseBeaconNode} from "./lighthouse.js";
 import {generateLodestarBeaconNode} from "./lodestar.js";
 
-export async function createCLNode<C extends CLClient>(
+type GeneratorRequiredOptions<C extends CLClient> = AtLeast<
+  CLClientGeneratorOptions<C>,
+  "id" | "paths" | "config" | "nodeIndex" | "genesisTime"
+> & {
+  genesisState?: BeaconStateAllForks;
+  runner: IRunner;
+};
+
+export async function createCLNode<B extends CLClient, V extends CLClient>({
+  beacon,
+  beaconOptions,
+  validator,
+  validatorOptions,
+}: {
+  beacon: B;
+  beaconOptions: GeneratorRequiredOptions<B>;
+  validator: V;
+  validatorOptions: GeneratorRequiredOptions<V>;
+}): Promise<CLNode> {
+  const beaconNode = await createCLNodeComponent(beacon, beaconOptions, "beacon");
+  const validatorNode = await createCLNodeComponent(validator, validatorOptions, "validator");
+
+  return {
+    ...beaconNode,
+    beaconJob: beaconNode.beaconJob,
+    validatorJob: validatorNode.validatorJob,
+  };
+}
+
+async function createCLNodeComponent<C extends CLClient>(
   client: C,
-  options: AtLeast<CLClientGeneratorOptions<C>, "id" | "paths" | "config" | "paths" | "nodeIndex" | "genesisTime"> & {
-    genesisState?: BeaconStateAllForks;
-    runner: IRunner;
-  }
+  options: GeneratorRequiredOptions<C>,
+  component: "beacon" | "validator"
 ): Promise<CLNode> {
   const {runner, config, genesisState} = options;
-  const clId = `${options.id}-cl-${client}`;
+  const clId = `${options.id}-cl-${client}-${component}`;
 
   const opts: CLClientGeneratorOptions = {
     ...options,
@@ -27,6 +54,8 @@ export async function createCLNode<C extends CLClient>(
     clientOptions: options.clientOptions ?? {},
     address: "127.0.0.1",
     engineUrls: options.engineUrls ?? [],
+    beacon: component === "beacon",
+    validator: component === "validator",
   };
 
   const metricServer = process.env.SIM_METRIC_SERVER_URL;

packages/cli/test/utils/simulation/cl_clients/lighthouse.ts

@@ -132,6 +132,9 @@ export const generateLighthouseBeaconNode: CLClientGenerator<CLClient.Lighthouse
     );
   }
 
+  const beaconJob = runner.create([{...beaconNodeJob, children: [...validatorClientsJobs]}]);
+  const validatorJob = runner.create(validatorClientsJobs);
+
   const httpClient = new HttpClient({baseUrl: `http://127.0.0.1:${httpPort}`});
   const api = getClient({baseUrl: `http://127.0.0.1:${httpPort}`}, {config}) as unknown as LighthouseAPI;
   api.lighthouse = {
@@ -147,7 +150,8 @@ export const generateLighthouseBeaconNode: CLClientGenerator<CLClient.Lighthouse
     keys,
     api,
     keyManager: keyManagerGetClient({baseUrl: `http://127.0.0.1:${keymanagerPort}`}, {config}),
-    job: runner.create([{...beaconNodeJob, children: [...validatorClientsJobs]}]),
+    beaconJob: opts.beacon ? beaconJob : undefined,
+    validatorJob: opts.validator ? validatorJob : undefined,
   };
 };
 

packages/cli/test/utils/simulation/cl_clients/lodestar.ts

@@ -82,7 +82,8 @@ export const generateLodestarBeaconNode: CLClientGenerator<CLClient.Lodestar> =
     );
   }
 
-  const job = runner.create([
+  const validatorJob = runner.create(validatorClientsJobs);
+  const beaconJob = runner.create([
     {
       id,
       bootstrap: async () => {
@@ -108,7 +109,6 @@ export const generateLodestarBeaconNode: CLClientGenerator<CLClient.Lodestar> =
           return {ok: false, reason: (err as Error).message, checkId: "eth/v1/node/health query"};
         }
       },
-      children: validatorClientsJobs,
     },
   ]);
 
@@ -119,7 +119,8 @@ export const generateLodestarBeaconNode: CLClientGenerator<CLClient.Lodestar> =
     keys,
     api: getClient({baseUrl: `http://127.0.0.1:${ports.cl.httpPort}`}, {config}),
     keyManager: keyManagerGetClient({baseUrl: `http://127.0.0.1:${ports.cl.keymanagerPort}`}, {config}),
-    job,
+    beaconJob: opts.beacon ? beaconJob : undefined,
+    validatorJob: opts.validator ? validatorJob : undefined,
   };
 };