feat: clean up options and startup sequence

packages/cli/src/cmds/bootnode/handler.ts

@@ -1,6 +1,6 @@
 import path from "node:path";
 import {Multiaddr, multiaddr} from "@multiformats/multiaddr";
-import {Discv5} from "@chainsafe/discv5";
+import {Discv5, ENR} from "@chainsafe/discv5";
 import {ErrorAborted} from "@lodestar/utils";
 import {HttpMetricsServer, RegistryMetricCreator, getHttpMetricsServer} from "@lodestar/beacon-node";
 
@@ -29,13 +29,10 @@ export async function bootnodeHandler(args: BootnodeArgs & GlobalArgs): Promise<
 
   logger.info("Lodestar Bootnode", {network, version, commit});
   logger.info("Bind address", bindAddrs);
-  const advertisedAddrs = Object.fromEntries(
-    [
-      ["ip4", enr.getLocationMultiaddr("udp4")?.toString()],
-      ["ip6", enr.getLocationMultiaddr("udp6")?.toString()],
-    ].filter(([_, v]) => v)
-  ) as Record<string, string>;
-  logger.info("Advertised address", advertisedAddrs);
+  logger.info("Advertised address", {
+    ip4: enr.getLocationMultiaddr("udp4")?.toString(),
+    ip6: enr.getLocationMultiaddr("udp6")?.toString(),
+  });
   logger.info("Identity", {peerId: peerId.toString(), nodeId: enr.nodeId});
   logger.info("ENR", {enr: enr.encodeTxt()});
 
@@ -63,16 +60,66 @@ export async function bootnodeHandler(args: BootnodeArgs & GlobalArgs): Promise<
       config: {enrUpdate: !enr.ip && !enr.ip6},
       metricsRegistry,
     });
-    for (const bootEnr of discv5Args.bootEnrs) {
+
+    // If there are any bootnodes, add them to the routing table
+    for (const bootEnrStr of Array.from(new Set(discv5Args.bootEnrs).values())) {
+      const bootEnr = ENR.decodeTxt(bootEnrStr);
+      logger.info("Adding bootnode", {
+        ip4: bootEnr.getLocationMultiaddr("udp4")?.toString(),
+        ip6: bootEnr.getLocationMultiaddr("udp6")?.toString(),
+        peerId: (await bootEnr.peerId()).toString(),
+        nodeId: enr.nodeId,
+      });
       discv5.addEnr(bootEnr);
     }
 
+    // start the server
+    await discv5.start();
+
+    // if there are peers in the local routing table, establish a session by running a query
+    if (discv5.kadValues().length) {
+      void discv5.findRandomNode();
+    }
+
+    discv5.on("multiaddrUpdated", (addr) => {
+      logger.info("Advertised socket address updated", {addr: addr.toString()});
+    });
+
+    // respond with metrics every 10 seconds
+    const printInterval = setInterval(() => {
+      let ip4Only = 0;
+      let ip6Only = 0;
+      let ip4ip6 = 0;
+      let unreachable = 0;
+      for (const kadEnr of discv5.kadValues()) {
+        const hasIp4 = kadEnr.getLocationMultiaddr("udp4");
+        const hasIp6 = kadEnr.getLocationMultiaddr("udp6");
+        if (hasIp4 && hasIp6) {
+          ip4ip6++;
+        } else if (hasIp4) {
+          ip4Only++;
+        } else if (hasIp6) {
+          ip6Only++;
+        } else {
+          unreachable++;
+        }
+      }
+      logger.info("Server metrics", {
+        connectedPeers: discv5.connectedPeerCount,
+        activeSessions: discv5.sessionService.sessionsSize(),
+        ip4Nodes: ip4Only,
+        ip6Nodes: ip6Only,
+        ip4AndIp6Nodes: ip4ip6,
+        unreachableNodes: unreachable,
+      });
+    }, 10_000);
+
     // Intercept SIGINT signal, to perform final ops before exiting
     onGracefulShutdown(async () => {
       if (args.persistNetworkIdentity) {
         try {
           const enrPath = path.join(bootnodeDir, "enr");
-          writeFile600Perm(enrPath, enr);
+          writeFile600Perm(enrPath, enr.encodeTxt());
         } catch (e) {
           logger.warn("Unable to persist enr", {}, e as Error);
         }
@@ -84,12 +131,12 @@ export async function bootnodeHandler(args: BootnodeArgs & GlobalArgs): Promise<
       "abort",
       async () => {
         try {
+          discv5.removeAllListeners();
+          clearInterval(printInterval);
+
           await metricsServer?.close();
           await discv5.stop();
           logger.debug("Bootnode closed");
-          // Explicitly exit until active handles issue is resolved
-          // See https://github.com/ChainSafe/lodestar/issues/5642
-          process.exit(0);
         } catch (e) {
           logger.error("Error closing bootnode", {}, e as Error);
           // Must explicitly exit process due to potential active handles

packages/cli/src/cmds/bootnode/index.ts

@@ -3,10 +3,10 @@ import {GlobalArgs} from "../../options/index.js";
 import {bootnodeOptions, BootnodeArgs} from "./options.js";
 import {bootnodeHandler} from "./handler.js";
 
-export const beacon: CliCommand<BootnodeArgs, GlobalArgs> = {
+export const bootnode: CliCommand<BootnodeArgs, GlobalArgs> = {
   command: "bootnode",
   describe:
-    "Start a discv5 bootnode. This will NOT perform any beacon node functions, rather, it will run a discv5 service that allows nodes on the network to discover one another.",
+    "Run a discv5 bootnode. This will NOT perform any beacon node functions, rather, it will run a discv5 service that allows nodes on the network to discover one another.",
   options: bootnodeOptions as CliCommandOptions<BootnodeArgs>,
   handler: bootnodeHandler,
 };

packages/cli/src/cmds/bootnode/options.ts

@@ -1,10 +1,15 @@
 import {Options} from "yargs";
 import {LogArgs, logOptions} from "../../options/logOptions.js";
 import {CliCommandOptions} from "../../util/index.js";
-import {NetworkArgs, options as networkOptions} from "../../options/beaconNodeOptions/network.js";
 import {MetricsArgs, options as metricsOptions} from "../../options/beaconNodeOptions/metrics.js";
+import {defaultListenAddress, defaultP2pPort, defaultP2pPort6} from "../../options/beaconNodeOptions/network.js";
 
 type BootnodeExtraArgs = {
+  listenAddress?: string;
+  port?: number;
+  listenAddress6?: string;
+  port6?: number;
+  bootnodes?: string[];
   bootnodesFile?: string;
   persistNetworkIdentity?: boolean;
   "enr.ip"?: string;
@@ -15,17 +20,57 @@ type BootnodeExtraArgs = {
 };
 
 export const bootnodeExtraOptions: CliCommandOptions<BootnodeExtraArgs> = {
+  listenAddress: {
+    type: "string",
+    description: "The IPv4 address to listen for discv5 connections",
+    defaultDescription: defaultListenAddress,
+    group: "network",
+  },
+
+  port: {
+    alias: "discoveryPort",
+    description: "The UDP port to listen on",
+    type: "number",
+    defaultDescription: String(defaultP2pPort),
+    group: "network",
+  },
+
+  listenAddress6: {
+    type: "string",
+    description: "The IPv6 address to listen for discv5 connections",
+    group: "network",
+  },
+
+  port6: {
+    alias: "discoveryPort6",
+    description: "The UDP port to listen on",
+    type: "number",
+    defaultDescription: String(defaultP2pPort6),
+    group: "network",
+  },
+
+  bootnodes: {
+    type: "array",
+    description: "Additional bootnodes for discv5 discovery",
+    defaultDescription: JSON.stringify([]),
+    // Each bootnode entry could be comma separated, just deserialize it into a single array
+    // as comma separated entries are generally most friendly in ansible kind of setups, i.e.
+    // [ "en1", "en2,en3" ] => [ 'en1', 'en2', 'en3' ]
+    coerce: (args: string[]) => args.map((item) => item.split(",")).flat(1),
+    group: "network",
+  },
+
   bootnodesFile: {
-    hidden: true,
-    description: "Bootnodes file path",
+    description: "Additional bootnodes for discv5 discovery file path",
     type: "string",
+    group: "network",
   },
 
   persistNetworkIdentity: {
-    hidden: true,
     description: "Whether to reuse the same peer-id across restarts",
     default: true,
     type: "boolean",
+    group: "network",
   },
 
   "enr.ip": {
@@ -50,16 +95,15 @@ export const bootnodeExtraOptions: CliCommandOptions<BootnodeExtraArgs> = {
   },
   nat: {
     type: "boolean",
-    description: "Allow configuration of non-local addresses",
+    description: "Allow ENR configuration of non-local addresses",
     group: "enr",
   },
 };
 
-export type BootnodeArgs = BootnodeExtraArgs & LogArgs & NetworkArgs & MetricsArgs;
+export type BootnodeArgs = BootnodeExtraArgs & LogArgs & MetricsArgs;
 
 export const bootnodeOptions: {[k: string]: Options} = {
   ...bootnodeExtraOptions,
   ...logOptions,
-  ...networkOptions,
   ...metricsOptions,
 };

packages/cli/src/cmds/index.ts

@@ -4,10 +4,12 @@ import {beacon} from "./beacon/index.js";
 import {dev} from "./dev/index.js";
 import {validator} from "./validator/index.js";
 import {lightclient} from "./lightclient/index.js";
+import {bootnode} from "./bootnode/index.js";
 
 export const cmds: Required<CliCommand<GlobalArgs, Record<never, never>>>["subcommands"] = [
   beacon,
   validator,
   lightclient,
   dev,
+  bootnode,
 ];

packages/cli/src/options/beaconNodeOptions/network.ts

@@ -3,7 +3,7 @@ import {ENR} from "@chainsafe/discv5";
 import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node";
 import {CliCommandOptions, YargsError} from "../../util/index.js";
 
-const defaultListenAddress = "0.0.0.0";
+export const defaultListenAddress = "0.0.0.0";
 export const defaultP2pPort = 9000;
 export const defaultP2pPort6 = 9090;