diff --git a/packages/cli/src/cmds/bootnode/handler.ts b/packages/cli/src/cmds/bootnode/handler.ts index 3bb216ff4e26..0623d67bda59 100644 --- a/packages/cli/src/cmds/bootnode/handler.ts +++ b/packages/cli/src/cmds/bootnode/handler.ts @@ -1,6 +1,6 @@ import path from "node:path"; import {Multiaddr, multiaddr} from "@multiformats/multiaddr"; -import {Discv5} from "@chainsafe/discv5"; +import {Discv5, ENR} from "@chainsafe/discv5"; import {ErrorAborted} from "@lodestar/utils"; import {HttpMetricsServer, RegistryMetricCreator, getHttpMetricsServer} from "@lodestar/beacon-node"; @@ -29,13 +29,10 @@ export async function bootnodeHandler(args: BootnodeArgs & GlobalArgs): Promise< logger.info("Lodestar Bootnode", {network, version, commit}); logger.info("Bind address", bindAddrs); - const advertisedAddrs = Object.fromEntries( - [ - ["ip4", enr.getLocationMultiaddr("udp4")?.toString()], - ["ip6", enr.getLocationMultiaddr("udp6")?.toString()], - ].filter(([_, v]) => v) - ) as Record; - logger.info("Advertised address", advertisedAddrs); + logger.info("Advertised address", { + ip4: enr.getLocationMultiaddr("udp4")?.toString(), + ip6: enr.getLocationMultiaddr("udp6")?.toString(), + }); logger.info("Identity", {peerId: peerId.toString(), nodeId: enr.nodeId}); logger.info("ENR", {enr: enr.encodeTxt()}); @@ -63,16 +60,66 @@ export async function bootnodeHandler(args: BootnodeArgs & GlobalArgs): Promise< config: {enrUpdate: !enr.ip && !enr.ip6}, metricsRegistry, }); - for (const bootEnr of discv5Args.bootEnrs) { + + // If there are any bootnodes, add them to the routing table + for (const bootEnrStr of Array.from(new Set(discv5Args.bootEnrs).values())) { + const bootEnr = ENR.decodeTxt(bootEnrStr); + logger.info("Adding bootnode", { + ip4: bootEnr.getLocationMultiaddr("udp4")?.toString(), + ip6: bootEnr.getLocationMultiaddr("udp6")?.toString(), + peerId: (await bootEnr.peerId()).toString(), + nodeId: enr.nodeId, + }); discv5.addEnr(bootEnr); } + // start the server + await discv5.start(); + + // if there are peers in the local routing table, establish a session by running a query + if (discv5.kadValues().length) { + void discv5.findRandomNode(); + } + + discv5.on("multiaddrUpdated", (addr) => { + logger.info("Advertised socket address updated", {addr: addr.toString()}); + }); + + // respond with metrics every 10 seconds + const printInterval = setInterval(() => { + let ip4Only = 0; + let ip6Only = 0; + let ip4ip6 = 0; + let unreachable = 0; + for (const kadEnr of discv5.kadValues()) { + const hasIp4 = kadEnr.getLocationMultiaddr("udp4"); + const hasIp6 = kadEnr.getLocationMultiaddr("udp6"); + if (hasIp4 && hasIp6) { + ip4ip6++; + } else if (hasIp4) { + ip4Only++; + } else if (hasIp6) { + ip6Only++; + } else { + unreachable++; + } + } + logger.info("Server metrics", { + connectedPeers: discv5.connectedPeerCount, + activeSessions: discv5.sessionService.sessionsSize(), + ip4Nodes: ip4Only, + ip6Nodes: ip6Only, + ip4AndIp6Nodes: ip4ip6, + unreachableNodes: unreachable, + }); + }, 10_000); + // Intercept SIGINT signal, to perform final ops before exiting onGracefulShutdown(async () => { if (args.persistNetworkIdentity) { try { const enrPath = path.join(bootnodeDir, "enr"); - writeFile600Perm(enrPath, enr); + writeFile600Perm(enrPath, enr.encodeTxt()); } catch (e) { logger.warn("Unable to persist enr", {}, e as Error); } @@ -84,12 +131,12 @@ export async function bootnodeHandler(args: BootnodeArgs & GlobalArgs): Promise< "abort", async () => { try { + discv5.removeAllListeners(); + clearInterval(printInterval); + await metricsServer?.close(); await discv5.stop(); logger.debug("Bootnode closed"); - // Explicitly exit until active handles issue is resolved - // See https://github.com/ChainSafe/lodestar/issues/5642 - process.exit(0); } catch (e) { logger.error("Error closing bootnode", {}, e as Error); // Must explicitly exit process due to potential active handles diff --git a/packages/cli/src/cmds/bootnode/index.ts b/packages/cli/src/cmds/bootnode/index.ts index 0e2bfd6bd456..c9a7db71eadc 100644 --- a/packages/cli/src/cmds/bootnode/index.ts +++ b/packages/cli/src/cmds/bootnode/index.ts @@ -3,10 +3,10 @@ import {GlobalArgs} from "../../options/index.js"; import {bootnodeOptions, BootnodeArgs} from "./options.js"; import {bootnodeHandler} from "./handler.js"; -export const beacon: CliCommand = { +export const bootnode: CliCommand = { command: "bootnode", describe: - "Start a discv5 bootnode. This will NOT perform any beacon node functions, rather, it will run a discv5 service that allows nodes on the network to discover one another.", + "Run a discv5 bootnode. This will NOT perform any beacon node functions, rather, it will run a discv5 service that allows nodes on the network to discover one another.", options: bootnodeOptions as CliCommandOptions, handler: bootnodeHandler, }; diff --git a/packages/cli/src/cmds/bootnode/options.ts b/packages/cli/src/cmds/bootnode/options.ts index 246dd383b572..622d7b2d506a 100644 --- a/packages/cli/src/cmds/bootnode/options.ts +++ b/packages/cli/src/cmds/bootnode/options.ts @@ -1,10 +1,15 @@ import {Options} from "yargs"; import {LogArgs, logOptions} from "../../options/logOptions.js"; import {CliCommandOptions} from "../../util/index.js"; -import {NetworkArgs, options as networkOptions} from "../../options/beaconNodeOptions/network.js"; import {MetricsArgs, options as metricsOptions} from "../../options/beaconNodeOptions/metrics.js"; +import {defaultListenAddress, defaultP2pPort, defaultP2pPort6} from "../../options/beaconNodeOptions/network.js"; type BootnodeExtraArgs = { + listenAddress?: string; + port?: number; + listenAddress6?: string; + port6?: number; + bootnodes?: string[]; bootnodesFile?: string; persistNetworkIdentity?: boolean; "enr.ip"?: string; @@ -15,17 +20,57 @@ type BootnodeExtraArgs = { }; export const bootnodeExtraOptions: CliCommandOptions = { + listenAddress: { + type: "string", + description: "The IPv4 address to listen for discv5 connections", + defaultDescription: defaultListenAddress, + group: "network", + }, + + port: { + alias: "discoveryPort", + description: "The UDP port to listen on", + type: "number", + defaultDescription: String(defaultP2pPort), + group: "network", + }, + + listenAddress6: { + type: "string", + description: "The IPv6 address to listen for discv5 connections", + group: "network", + }, + + port6: { + alias: "discoveryPort6", + description: "The UDP port to listen on", + type: "number", + defaultDescription: String(defaultP2pPort6), + group: "network", + }, + + bootnodes: { + type: "array", + description: "Additional bootnodes for discv5 discovery", + defaultDescription: JSON.stringify([]), + // Each bootnode entry could be comma separated, just deserialize it into a single array + // as comma separated entries are generally most friendly in ansible kind of setups, i.e. + // [ "en1", "en2,en3" ] => [ 'en1', 'en2', 'en3' ] + coerce: (args: string[]) => args.map((item) => item.split(",")).flat(1), + group: "network", + }, + bootnodesFile: { - hidden: true, - description: "Bootnodes file path", + description: "Additional bootnodes for discv5 discovery file path", type: "string", + group: "network", }, persistNetworkIdentity: { - hidden: true, description: "Whether to reuse the same peer-id across restarts", default: true, type: "boolean", + group: "network", }, "enr.ip": { @@ -50,16 +95,15 @@ export const bootnodeExtraOptions: CliCommandOptions = { }, nat: { type: "boolean", - description: "Allow configuration of non-local addresses", + description: "Allow ENR configuration of non-local addresses", group: "enr", }, }; -export type BootnodeArgs = BootnodeExtraArgs & LogArgs & NetworkArgs & MetricsArgs; +export type BootnodeArgs = BootnodeExtraArgs & LogArgs & MetricsArgs; export const bootnodeOptions: {[k: string]: Options} = { ...bootnodeExtraOptions, ...logOptions, - ...networkOptions, ...metricsOptions, }; diff --git a/packages/cli/src/cmds/index.ts b/packages/cli/src/cmds/index.ts index b9aef4544711..849cb23d9af7 100644 --- a/packages/cli/src/cmds/index.ts +++ b/packages/cli/src/cmds/index.ts @@ -4,10 +4,12 @@ import {beacon} from "./beacon/index.js"; import {dev} from "./dev/index.js"; import {validator} from "./validator/index.js"; import {lightclient} from "./lightclient/index.js"; +import {bootnode} from "./bootnode/index.js"; export const cmds: Required>>["subcommands"] = [ beacon, validator, lightclient, dev, + bootnode, ]; diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 06a46cd849e2..8bbe65065965 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -3,7 +3,7 @@ import {ENR} from "@chainsafe/discv5"; import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; import {CliCommandOptions, YargsError} from "../../util/index.js"; -const defaultListenAddress = "0.0.0.0"; +export const defaultListenAddress = "0.0.0.0"; export const defaultP2pPort = 9000; export const defaultP2pPort6 = 9090;