diff --git a/packages/cli/src/cmds/beacon/handler.ts b/packages/cli/src/cmds/beacon/handler.ts index c4b981f90c08..0d7fd8b4521a 100644 --- a/packages/cli/src/cmds/beacon/handler.ts +++ b/packages/cli/src/cmds/beacon/handler.ts @@ -20,6 +20,7 @@ import { pruneOldFilesInDir, } from "../../util/index.js"; import {getVersionData} from "../../util/version.js"; +import {LogArgs} from "../../options/logOptions.js"; import {BeaconArgs} from "./options.js"; import {getBeaconPaths} from "./paths.js"; import {initBeaconState} from "./initBeaconState.js"; @@ -205,8 +206,13 @@ export async function beaconHandlerInit(args: BeaconArgs & GlobalArgs) { return {config, options, beaconPaths, network, version, commit, peerId, logger}; } -export function initLogger(args: BeaconArgs, dataDir: string, config: ChainForkConfig): LoggerNode { - const defaultLogFilepath = path.join(dataDir, "beacon.log"); +export function initLogger( + args: LogArgs & Pick, + dataDir: string, + config: ChainForkConfig, + fileName = "beacon.log" +): LoggerNode { + const defaultLogFilepath = path.join(dataDir, fileName); const logger = getNodeLogger(parseLoggerArgs(args, {defaultLogFilepath}, config)); try { cleanOldLogFiles(args, {defaultLogFilepath}); diff --git a/packages/cli/src/cmds/bootnode/handler.ts b/packages/cli/src/cmds/bootnode/handler.ts new file mode 100644 index 000000000000..0623d67bda59 --- /dev/null +++ b/packages/cli/src/cmds/bootnode/handler.ts @@ -0,0 +1,186 @@ +import path from "node:path"; +import {Multiaddr, multiaddr} from "@multiformats/multiaddr"; +import {Discv5, ENR} from "@chainsafe/discv5"; +import {ErrorAborted} from "@lodestar/utils"; +import {HttpMetricsServer, RegistryMetricCreator, getHttpMetricsServer} from "@lodestar/beacon-node"; + +import {GlobalArgs} from "../../options/index.js"; +import {getBeaconConfigFromArgs} from "../../config/index.js"; +import {getNetworkBootnodes, isKnownNetworkName, readBootnodes} from "../../networks/index.js"; +import {onGracefulShutdown, mkdir, writeFile600Perm} from "../../util/index.js"; +import {getVersionData} from "../../util/version.js"; +import {initPeerIdAndEnr} from "../beacon/initPeerIdAndEnr.js"; +import {parseArgs as parseMetricsArgs} from "../../options/beaconNodeOptions/metrics.js"; +import {parseArgs as parseNetworkArgs} from "../../options/beaconNodeOptions/network.js"; +import {getBeaconPaths} from "../beacon/paths.js"; +import {BeaconArgs} from "../beacon/options.js"; +import {initLogger} from "../beacon/handler.js"; +import {BootnodeArgs} from "./options.js"; + +/** + * Runs a bootnode. + */ +export async function bootnodeHandler(args: BootnodeArgs & GlobalArgs): Promise { + const {discv5Args, metricsArgs, bootnodeDir, network, version, commit, peerId, enr, logger} = + await bootnodeHandlerInit(args); + + const abortController = new AbortController(); + const bindAddrs = discv5Args.bindAddrs; + + logger.info("Lodestar Bootnode", {network, version, commit}); + logger.info("Bind address", bindAddrs); + logger.info("Advertised address", { + ip4: enr.getLocationMultiaddr("udp4")?.toString(), + ip6: enr.getLocationMultiaddr("udp6")?.toString(), + }); + logger.info("Identity", {peerId: peerId.toString(), nodeId: enr.nodeId}); + logger.info("ENR", {enr: enr.encodeTxt()}); + + // bootnode setup + try { + let metricsRegistry: RegistryMetricCreator | undefined; + let metricsServer: HttpMetricsServer | undefined; + if (metricsArgs.enabled) { + metricsRegistry = new RegistryMetricCreator(); + metricsRegistry.static({ + name: "bootnode_version", + help: "Bootnode version", + value: {version, commit, network}, + }); + metricsServer = await getHttpMetricsServer(metricsArgs, {register: metricsRegistry, logger}); + } + + const discv5 = Discv5.create({ + enr, + peerId, + bindAddrs: { + ip4: (bindAddrs.ip4 ? multiaddr(bindAddrs.ip4) : undefined) as Multiaddr, + ip6: bindAddrs.ip6 ? multiaddr(bindAddrs.ip6) : undefined, + }, + config: {enrUpdate: !enr.ip && !enr.ip6}, + metricsRegistry, + }); + + // If there are any bootnodes, add them to the routing table + for (const bootEnrStr of Array.from(new Set(discv5Args.bootEnrs).values())) { + const bootEnr = ENR.decodeTxt(bootEnrStr); + logger.info("Adding bootnode", { + ip4: bootEnr.getLocationMultiaddr("udp4")?.toString(), + ip6: bootEnr.getLocationMultiaddr("udp6")?.toString(), + peerId: (await bootEnr.peerId()).toString(), + nodeId: enr.nodeId, + }); + discv5.addEnr(bootEnr); + } + + // start the server + await discv5.start(); + + // if there are peers in the local routing table, establish a session by running a query + if (discv5.kadValues().length) { + void discv5.findRandomNode(); + } + + discv5.on("multiaddrUpdated", (addr) => { + logger.info("Advertised socket address updated", {addr: addr.toString()}); + }); + + // respond with metrics every 10 seconds + const printInterval = setInterval(() => { + let ip4Only = 0; + let ip6Only = 0; + let ip4ip6 = 0; + let unreachable = 0; + for (const kadEnr of discv5.kadValues()) { + const hasIp4 = kadEnr.getLocationMultiaddr("udp4"); + const hasIp6 = kadEnr.getLocationMultiaddr("udp6"); + if (hasIp4 && hasIp6) { + ip4ip6++; + } else if (hasIp4) { + ip4Only++; + } else if (hasIp6) { + ip6Only++; + } else { + unreachable++; + } + } + logger.info("Server metrics", { + connectedPeers: discv5.connectedPeerCount, + activeSessions: discv5.sessionService.sessionsSize(), + ip4Nodes: ip4Only, + ip6Nodes: ip6Only, + ip4AndIp6Nodes: ip4ip6, + unreachableNodes: unreachable, + }); + }, 10_000); + + // Intercept SIGINT signal, to perform final ops before exiting + onGracefulShutdown(async () => { + if (args.persistNetworkIdentity) { + try { + const enrPath = path.join(bootnodeDir, "enr"); + writeFile600Perm(enrPath, enr.encodeTxt()); + } catch (e) { + logger.warn("Unable to persist enr", {}, e as Error); + } + } + abortController.abort(); + }, logger.info.bind(logger)); + + abortController.signal.addEventListener( + "abort", + async () => { + try { + discv5.removeAllListeners(); + clearInterval(printInterval); + + await metricsServer?.close(); + await discv5.stop(); + logger.debug("Bootnode closed"); + } catch (e) { + logger.error("Error closing bootnode", {}, e as Error); + // Must explicitly exit process due to potential active handles + process.exit(1); + } + }, + {once: true} + ); + } catch (e) { + if (e instanceof ErrorAborted) { + logger.info(e.message); // Let the user know the abort was received but don't print as error + } else { + throw e; + } + } +} + +/** Separate function to simplify unit testing of options merging */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export async function bootnodeHandlerInit(args: BootnodeArgs & GlobalArgs) { + const {config, network} = getBeaconConfigFromArgs(args); + const {version, commit} = getVersionData(); + const beaconPaths = getBeaconPaths(args, network); + // Use a separate directory to store bootnode enr + peer-id + const bootnodeDir = path.join(beaconPaths.dataDir, "bootnode"); + const {discv5: discv5Args} = parseNetworkArgs(args); + const metricsArgs = parseMetricsArgs(args); + if (!discv5Args) { + // Unreachable because bootnode requires discv5 to be enabled - duh + throw new Error("unreachable - bootnode requires discv5 to be enabled"); + } + + // initialize directories + mkdir(beaconPaths.dataDir); + mkdir(bootnodeDir); + + // Fetch extra bootnodes + discv5Args.bootEnrs = (discv5Args.bootEnrs ?? []).concat( + args.bootnodesFile ? readBootnodes(args.bootnodesFile) : [], + isKnownNetworkName(network) ? await getNetworkBootnodes(network) : [] + ); + + const logger = initLogger(args, beaconPaths.dataDir, config, "bootnode.log"); + const {peerId, enr} = await initPeerIdAndEnr(args as unknown as BeaconArgs, bootnodeDir, logger); + + return {discv5Args, metricsArgs, bootnodeDir, network, version, commit, peerId, enr, logger}; +} diff --git a/packages/cli/src/cmds/bootnode/index.ts b/packages/cli/src/cmds/bootnode/index.ts new file mode 100644 index 000000000000..c9a7db71eadc --- /dev/null +++ b/packages/cli/src/cmds/bootnode/index.ts @@ -0,0 +1,12 @@ +import {CliCommand, CliCommandOptions} from "../../util/index.js"; +import {GlobalArgs} from "../../options/index.js"; +import {bootnodeOptions, BootnodeArgs} from "./options.js"; +import {bootnodeHandler} from "./handler.js"; + +export const bootnode: CliCommand = { + command: "bootnode", + describe: + "Run a discv5 bootnode. This will NOT perform any beacon node functions, rather, it will run a discv5 service that allows nodes on the network to discover one another.", + options: bootnodeOptions as CliCommandOptions, + handler: bootnodeHandler, +}; diff --git a/packages/cli/src/cmds/bootnode/options.ts b/packages/cli/src/cmds/bootnode/options.ts new file mode 100644 index 000000000000..622d7b2d506a --- /dev/null +++ b/packages/cli/src/cmds/bootnode/options.ts @@ -0,0 +1,109 @@ +import {Options} from "yargs"; +import {LogArgs, logOptions} from "../../options/logOptions.js"; +import {CliCommandOptions} from "../../util/index.js"; +import {MetricsArgs, options as metricsOptions} from "../../options/beaconNodeOptions/metrics.js"; +import {defaultListenAddress, defaultP2pPort, defaultP2pPort6} from "../../options/beaconNodeOptions/network.js"; + +type BootnodeExtraArgs = { + listenAddress?: string; + port?: number; + listenAddress6?: string; + port6?: number; + bootnodes?: string[]; + bootnodesFile?: string; + persistNetworkIdentity?: boolean; + "enr.ip"?: string; + "enr.ip6"?: string; + "enr.udp"?: number; + "enr.udp6"?: number; + nat?: boolean; +}; + +export const bootnodeExtraOptions: CliCommandOptions = { + listenAddress: { + type: "string", + description: "The IPv4 address to listen for discv5 connections", + defaultDescription: defaultListenAddress, + group: "network", + }, + + port: { + alias: "discoveryPort", + description: "The UDP port to listen on", + type: "number", + defaultDescription: String(defaultP2pPort), + group: "network", + }, + + listenAddress6: { + type: "string", + description: "The IPv6 address to listen for discv5 connections", + group: "network", + }, + + port6: { + alias: "discoveryPort6", + description: "The UDP port to listen on", + type: "number", + defaultDescription: String(defaultP2pPort6), + group: "network", + }, + + bootnodes: { + type: "array", + description: "Additional bootnodes for discv5 discovery", + defaultDescription: JSON.stringify([]), + // Each bootnode entry could be comma separated, just deserialize it into a single array + // as comma separated entries are generally most friendly in ansible kind of setups, i.e. + // [ "en1", "en2,en3" ] => [ 'en1', 'en2', 'en3' ] + coerce: (args: string[]) => args.map((item) => item.split(",")).flat(1), + group: "network", + }, + + bootnodesFile: { + description: "Additional bootnodes for discv5 discovery file path", + type: "string", + group: "network", + }, + + persistNetworkIdentity: { + description: "Whether to reuse the same peer-id across restarts", + default: true, + type: "boolean", + group: "network", + }, + + "enr.ip": { + description: "Override ENR IP entry", + type: "string", + group: "enr", + }, + "enr.udp": { + description: "Override ENR UDP entry", + type: "number", + group: "enr", + }, + "enr.ip6": { + description: "Override ENR IPv6 entry", + type: "string", + group: "enr", + }, + "enr.udp6": { + description: "Override ENR (IPv6-specific) UDP entry", + type: "number", + group: "enr", + }, + nat: { + type: "boolean", + description: "Allow ENR configuration of non-local addresses", + group: "enr", + }, +}; + +export type BootnodeArgs = BootnodeExtraArgs & LogArgs & MetricsArgs; + +export const bootnodeOptions: {[k: string]: Options} = { + ...bootnodeExtraOptions, + ...logOptions, + ...metricsOptions, +}; diff --git a/packages/cli/src/cmds/index.ts b/packages/cli/src/cmds/index.ts index b9aef4544711..849cb23d9af7 100644 --- a/packages/cli/src/cmds/index.ts +++ b/packages/cli/src/cmds/index.ts @@ -4,10 +4,12 @@ import {beacon} from "./beacon/index.js"; import {dev} from "./dev/index.js"; import {validator} from "./validator/index.js"; import {lightclient} from "./lightclient/index.js"; +import {bootnode} from "./bootnode/index.js"; export const cmds: Required>>["subcommands"] = [ beacon, validator, lightclient, dev, + bootnode, ]; diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 06a46cd849e2..8bbe65065965 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -3,7 +3,7 @@ import {ENR} from "@chainsafe/discv5"; import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; import {CliCommandOptions, YargsError} from "../../util/index.js"; -const defaultListenAddress = "0.0.0.0"; +export const defaultListenAddress = "0.0.0.0"; export const defaultP2pPort = 9000; export const defaultP2pPort6 = 9090;