From c78b6cee36b5f3b0fda9e0ac11fad350af7341a1 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Thu, 18 Jul 2024 16:11:52 +0500 Subject: [PATCH 01/11] feat: add cli flag to disable light client server (#6913) * Add disable flag * Add util function to check lightclient server * Add reqresp checks * Add api checks * Fix beacon-node usage of lightclient server * Fix types * Fix the code with feedback * Disable gossipsub topic for lightclient * Fix the types * Update the type checks * Clean reassingment * Update code --- .../src/api/impl/lightclient/index.ts | 20 ++++++++++++++----- .../src/chain/archiver/archiveBlocks.ts | 6 ++++-- .../src/chain/blocks/importBlock.ts | 2 +- packages/beacon-node/src/chain/chain.ts | 7 ++++--- packages/beacon-node/src/chain/interface.ts | 2 +- .../src/chain/lightClient/index.ts | 1 + .../validation/lightClientFinalityUpdate.ts | 3 +++ .../validation/lightClientOptimisticUpdate.ts | 3 +++ .../src/network/core/networkCore.ts | 14 +++++++++---- .../src/network/gossip/gossipsub.ts | 12 ++++++++--- .../beacon-node/src/network/gossip/topic.ts | 8 +++++--- packages/beacon-node/src/network/options.ts | 7 +++++-- .../src/network/reqresp/ReqRespBeaconNode.ts | 6 ++++-- .../reqresp/handlers/lightClientBootstrap.ts | 3 +++ .../handlers/lightClientFinalityUpdate.ts | 3 +++ .../handlers/lightClientOptimisticUpdate.ts | 3 +++ .../handlers/lightClientUpdatesByRange.ts | 3 +++ .../beacon-node/src/node/utils/lightclient.ts | 7 +++++++ .../lightClientFinalityUpdate.test.ts | 3 ++- .../lightClientOptimisticUpdate.test.ts | 3 ++- packages/cli/src/cmds/beacon/handler.ts | 4 ++++ packages/cli/src/cmds/beacon/options.ts | 6 ++++++ packages/utils/src/index.ts | 2 +- packages/utils/src/types.ts | 7 +++++++ 24 files changed, 106 insertions(+), 29 deletions(-) create mode 100644 packages/beacon-node/src/node/utils/lightclient.ts diff --git a/packages/beacon-node/src/api/impl/lightclient/index.ts b/packages/beacon-node/src/api/impl/lightclient/index.ts index 13deb16a9cb0..0dd366accf68 100644 --- a/packages/beacon-node/src/api/impl/lightclient/index.ts +++ b/packages/beacon-node/src/api/impl/lightclient/index.ts @@ -3,7 +3,7 @@ import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; import {MAX_REQUEST_LIGHT_CLIENT_UPDATES, MAX_REQUEST_LIGHT_CLIENT_COMMITTEE_HASHES} from "@lodestar/params"; import {ApiModules} from "../types.js"; - +import {assertLightClientServer} from "../../../node/utils/lightclient.js"; // TODO: Import from lightclient/server package export function getLightclientApi({ @@ -12,9 +12,12 @@ export function getLightclientApi({ }: Pick): ApplicationMethods { return { async getLightClientUpdatesByRange({startPeriod, count}) { + const lightClientServer = chain.lightClientServer; + assertLightClientServer(lightClientServer); + const maxAllowedCount = Math.min(MAX_REQUEST_LIGHT_CLIENT_UPDATES, count); const periods = Array.from({length: maxAllowedCount}, (_ignored, i) => i + startPeriod); - const updates = await Promise.all(periods.map((period) => chain.lightClientServer.getUpdate(period))); + const updates = await Promise.all(periods.map((period) => lightClientServer.getUpdate(period))); return { data: updates, meta: {versions: updates.map((update) => config.getForkName(update.attestedHeader.beacon.slot))}, @@ -22,6 +25,8 @@ export function getLightclientApi({ }, async getLightClientOptimisticUpdate() { + assertLightClientServer(chain.lightClientServer); + const update = chain.lightClientServer.getOptimisticUpdate(); if (update === null) { throw Error("No optimistic update available"); @@ -30,6 +35,8 @@ export function getLightclientApi({ }, async getLightClientFinalityUpdate() { + assertLightClientServer(chain.lightClientServer); + const update = chain.lightClientServer.getFinalityUpdate(); if (update === null) { throw Error("No finality update available"); @@ -38,16 +45,19 @@ export function getLightclientApi({ }, async getLightClientBootstrap({blockRoot}) { + assertLightClientServer(chain.lightClientServer); + const bootstrapProof = await chain.lightClientServer.getBootstrap(fromHex(blockRoot)); return {data: bootstrapProof, meta: {version: config.getForkName(bootstrapProof.header.beacon.slot)}}; }, async getLightClientCommitteeRoot({startPeriod, count}) { + const lightClientServer = chain.lightClientServer; + assertLightClientServer(lightClientServer); + const maxAllowedCount = Math.min(MAX_REQUEST_LIGHT_CLIENT_COMMITTEE_HASHES, count); const periods = Array.from({length: maxAllowedCount}, (_ignored, i) => i + startPeriod); - const committeeHashes = await Promise.all( - periods.map((period) => chain.lightClientServer.getCommitteeRoot(period)) - ); + const committeeHashes = await Promise.all(periods.map((period) => lightClientServer.getCommitteeRoot(period))); return {data: committeeHashes}; }, }; diff --git a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts index 71fa9fb61b2e..76bfe651ad77 100644 --- a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts @@ -32,7 +32,7 @@ export async function archiveBlocks( config: ChainForkConfig, db: IBeaconDb, forkChoice: IForkChoice, - lightclientServer: LightClientServer, + lightclientServer: LightClientServer | undefined, logger: Logger, finalizedCheckpoint: CheckpointHex, currentEpoch: Epoch, @@ -111,7 +111,9 @@ export async function archiveBlocks( nonCheckpointBlockRoots.push(block.root); } - await lightclientServer.pruneNonCheckpointData(nonCheckpointBlockRoots); + if (lightclientServer) { + await lightclientServer.pruneNonCheckpointData(nonCheckpointBlockRoots); + } logger.verbose("Archiving of finalized blocks complete", { totalArchived: finalizedCanonicalBlocks.length, diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index e67b6d1e9dbc..906be51434c2 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -305,7 +305,7 @@ export async function importBlock( // we want to import block asap so do this in the next event loop callInNextEventLoop(() => { try { - this.lightClientServer.onImportBlockHead( + this.lightClientServer?.onImportBlockHead( block.message as BeaconBlock, postState as CachedBeaconStateAltair, parentBlockSlot diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index a6912d952b68..61f044d4f895 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -126,7 +126,7 @@ export class BeaconChain implements IBeaconChain { readonly clock: IClock; readonly emitter: ChainEventEmitter; readonly regen: QueuedStateRegenerator; - readonly lightClientServer: LightClientServer; + readonly lightClientServer?: LightClientServer; readonly reprocessController: ReprocessController; // Ops pool @@ -305,7 +305,9 @@ export class BeaconChain implements IBeaconChain { signal, }); - const lightClientServer = new LightClientServer(opts, {config, db, metrics, emitter, logger}); + if (!opts.disableLightClientServer) { + this.lightClientServer = new LightClientServer(opts, {config, db, metrics, emitter, logger}); + } this.reprocessController = new ReprocessController(this.metrics); @@ -316,7 +318,6 @@ export class BeaconChain implements IBeaconChain { this.regen = regen; this.bls = bls; this.emitter = emitter; - this.lightClientServer = lightClientServer; this.archiver = new Archiver(db, this, logger, signal, opts); // always run PrepareNextSlotScheduler except for fork_choice spec tests diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index e412d8e8aafa..aa70b0bdcce4 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -95,7 +95,7 @@ export interface IBeaconChain { readonly clock: IClock; readonly emitter: ChainEventEmitter; readonly regen: IStateRegenerator; - readonly lightClientServer: LightClientServer; + readonly lightClientServer?: LightClientServer; readonly reprocessController: ReprocessController; readonly pubkey2index: PubkeyIndexMap; readonly index2pubkey: Index2PubkeyCache; diff --git a/packages/beacon-node/src/chain/lightClient/index.ts b/packages/beacon-node/src/chain/lightClient/index.ts index 6d713109f945..0230ca48c8be 100644 --- a/packages/beacon-node/src/chain/lightClient/index.ts +++ b/packages/beacon-node/src/chain/lightClient/index.ts @@ -60,6 +60,7 @@ import { export type LightClientServerOpts = { disableLightClientServerOnImportBlockHead?: boolean; + disableLightClientServer?: boolean; }; type DependentRootHex = RootHex; diff --git a/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts b/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts index 23b91dba5fd9..f4865c73f8e4 100644 --- a/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts +++ b/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts @@ -3,6 +3,7 @@ import {LightClientFinalityUpdate} from "@lodestar/types"; import {IBeaconChain} from "../interface.js"; import {LightClientError, LightClientErrorCode} from "../errors/lightClientError.js"; import {GossipAction} from "../errors/index.js"; +import {assertLightClientServer} from "../../node/utils/lightclient.js"; import {updateReceivedTooEarly} from "./lightClientOptimisticUpdate.js"; // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#light_client_finality_update @@ -11,6 +12,8 @@ export function validateLightClientFinalityUpdate( chain: IBeaconChain, gossipedFinalityUpdate: LightClientFinalityUpdate ): void { + assertLightClientServer(chain.lightClientServer); + // [IGNORE] No other finality_update with a lower or equal finalized_header.slot was already forwarded on the network const gossipedFinalitySlot = gossipedFinalityUpdate.finalizedHeader.beacon.slot; const localFinalityUpdate = chain.lightClientServer.getFinalityUpdate(); diff --git a/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts b/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts index 54b69f56808c..182321984af5 100644 --- a/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts +++ b/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts @@ -5,6 +5,7 @@ import {IBeaconChain} from "../interface.js"; import {LightClientError, LightClientErrorCode} from "../errors/lightClientError.js"; import {GossipAction} from "../errors/index.js"; import {MAXIMUM_GOSSIP_CLOCK_DISPARITY} from "../../constants/index.js"; +import {assertLightClientServer} from "../../node/utils/lightclient.js"; // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#light_client_optimistic_update export function validateLightClientOptimisticUpdate( @@ -12,6 +13,8 @@ export function validateLightClientOptimisticUpdate( chain: IBeaconChain, gossipedOptimisticUpdate: LightClientOptimisticUpdate ): void { + assertLightClientServer(chain.lightClientServer); + // [IGNORE] No other optimistic_update with a lower or equal attested_header.slot was already forwarded on the network const gossipedAttestedSlot = gossipedOptimisticUpdate.attestedHeader.beacon.slot; const localOptimisticUpdate = chain.lightClientServer.getOptimisticUpdate(); diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts index fdd2b681602f..07b346bc29e4 100644 --- a/packages/beacon-node/src/network/core/networkCore.ts +++ b/packages/beacon-node/src/network/core/networkCore.ts @@ -499,9 +499,12 @@ export class NetworkCore implements INetworkCore { private subscribeCoreTopicsAtFork(fork: ForkName): void { if (this.subscribedForks.has(fork)) return; this.subscribedForks.add(fork); - const {subscribeAllSubnets} = this.opts; + const {subscribeAllSubnets, disableLightClientServer} = this.opts; - for (const topic of getCoreTopicsAtFork(fork, {subscribeAllSubnets})) { + for (const topic of getCoreTopicsAtFork(fork, { + subscribeAllSubnets, + disableLightClientServer, + })) { this.gossip.subscribeTopic({...topic, fork}); } } @@ -509,9 +512,12 @@ export class NetworkCore implements INetworkCore { private unsubscribeCoreTopicsAtFork(fork: ForkName): void { if (!this.subscribedForks.has(fork)) return; this.subscribedForks.delete(fork); - const {subscribeAllSubnets} = this.opts; + const {subscribeAllSubnets, disableLightClientServer} = this.opts; - for (const topic of getCoreTopicsAtFork(fork, {subscribeAllSubnets})) { + for (const topic of getCoreTopicsAtFork(fork, { + subscribeAllSubnets, + disableLightClientServer, + })) { this.gossip.unsubscribeTopic({...topic, fork}); } } diff --git a/packages/beacon-node/src/network/gossip/gossipsub.ts b/packages/beacon-node/src/network/gossip/gossipsub.ts index 4066452f1e3b..e6977abe8ce6 100644 --- a/packages/beacon-node/src/network/gossip/gossipsub.ts +++ b/packages/beacon-node/src/network/gossip/gossipsub.ts @@ -56,6 +56,7 @@ export type Eth2GossipsubOpts = { gossipsubAwaitHandler?: boolean; disableFloodPublish?: boolean; skipParamsLog?: boolean; + disableLightClientServer?: boolean; }; /** @@ -124,7 +125,9 @@ export class Eth2Gossipsub extends GossipSub { isFinite(config.BELLATRIX_FORK_EPOCH) ? GOSSIP_MAX_SIZE_BELLATRIX : GOSSIP_MAX_SIZE ), metricsRegister: metricsRegister as MetricsRegister | null, - metricsTopicStrToLabel: metricsRegister ? getMetricsTopicStrToLabel(config) : undefined, + metricsTopicStrToLabel: metricsRegister + ? getMetricsTopicStrToLabel(config, {disableLightClientServer: opts.disableLightClientServer ?? false}) + : undefined, asyncValidation: true, maxOutboundBufferSize: MAX_OUTBOUND_BUFFER_SIZE, @@ -321,11 +324,14 @@ function attSubnetLabel(subnet: number): string { else return `0${subnet}`; } -function getMetricsTopicStrToLabel(config: BeaconConfig): TopicStrToLabel { +function getMetricsTopicStrToLabel(config: BeaconConfig, opts: {disableLightClientServer: boolean}): TopicStrToLabel { const metricsTopicStrToLabel = new Map(); for (const {name: fork} of config.forksAscendingEpochOrder) { - const topics = getCoreTopicsAtFork(fork, {subscribeAllSubnets: true}); + const topics = getCoreTopicsAtFork(fork, { + subscribeAllSubnets: true, + disableLightClientServer: opts.disableLightClientServer, + }); for (const topic of topics) { metricsTopicStrToLabel.set(stringifyGossipTopic(config, {...topic, fork}), topic.type); } diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index 0f3f6942a10a..4923b71e6887 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -201,7 +201,7 @@ export function parseGossipTopic(forkDigestContext: ForkDigestContext, topicStr: */ export function getCoreTopicsAtFork( fork: ForkName, - opts: {subscribeAllSubnets?: boolean} + opts: {subscribeAllSubnets?: boolean; disableLightClientServer?: boolean} ): GossipTopicTypeMap[keyof GossipTopicTypeMap][] { // Common topics for all forks const topics: GossipTopicTypeMap[keyof GossipTopicTypeMap][] = [ @@ -227,8 +227,10 @@ export function getCoreTopicsAtFork( // Any fork after altair included if (ForkSeq[fork] >= ForkSeq.altair) { topics.push({type: GossipType.sync_committee_contribution_and_proof}); - topics.push({type: GossipType.light_client_optimistic_update}); - topics.push({type: GossipType.light_client_finality_update}); + if (!opts.disableLightClientServer) { + topics.push({type: GossipType.light_client_optimistic_update}); + topics.push({type: GossipType.light_client_finality_update}); + } } if (opts.subscribeAllSubnets) { diff --git a/packages/beacon-node/src/network/options.ts b/packages/beacon-node/src/network/options.ts index 713ca4b21ad0..d2070873261b 100644 --- a/packages/beacon-node/src/network/options.ts +++ b/packages/beacon-node/src/network/options.ts @@ -8,11 +8,11 @@ import {SubnetsServiceOpts} from "./subnets/interface.js"; export interface NetworkOptions extends PeerManagerOpts, // remove all Functions - Omit, + Omit, NetworkProcessorOpts, PeerRpcScoreOpts, SubnetsServiceOpts, - Eth2GossipsubOpts { + Omit { localMultiaddrs: string[]; bootMultiaddrs?: string[]; subscribeAllSubnets?: boolean; @@ -22,6 +22,7 @@ export interface NetworkOptions private?: boolean; useWorker?: boolean; maxYoungGenerationSizeMb?: number; + disableLightClientServer?: boolean; } export const defaultNetworkOptions: NetworkOptions = { @@ -41,4 +42,6 @@ export const defaultNetworkOptions: NetworkOptions = { slotsToSubscribeBeforeAggregatorDuty: 2, // this should only be set to true if useWorker is true beaconAttestationBatchValidation: true, + // This will enable the light client server by default + disableLightClientServer: false, }; diff --git a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts index 8d131e9e9945..d34b379f4ccd 100644 --- a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts +++ b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts @@ -51,7 +51,7 @@ export interface ReqRespBeaconNodeModules { getHandler: GetReqRespHandlerFn; } -export type ReqRespBeaconNodeOpts = ReqRespOpts; +export type ReqRespBeaconNodeOpts = ReqRespOpts & {disableLightClientServer?: boolean}; /** * Implementation of Ethereum Consensus p2p Req/Resp domain. @@ -72,6 +72,7 @@ export class ReqRespBeaconNode extends ReqResp { private readonly config: BeaconConfig; protected readonly logger: Logger; + protected readonly disableLightClientServer: boolean; constructor(modules: ReqRespBeaconNodeModules, options: ReqRespBeaconNodeOpts = {}) { const {events, peersData, peerRpcScores, metadata, metrics, logger} = modules; @@ -95,6 +96,7 @@ export class ReqRespBeaconNode extends ReqResp { } ); + this.disableLightClientServer = options.disableLightClientServer ?? false; this.peerRpcScores = peerRpcScores; this.peersData = peersData; this.config = modules.config; @@ -233,7 +235,7 @@ export class ReqRespBeaconNode extends ReqResp { ); } - if (ForkSeq[fork] >= ForkSeq.altair) { + if (ForkSeq[fork] >= ForkSeq.altair && !this.disableLightClientServer) { // Should be okay to enable before altair, but for consistency only enable afterwards protocolsAtFork.push( [protocols.LightClientBootstrap(this.config), this.getHandler(ReqRespMethod.LightClientBootstrap)], diff --git a/packages/beacon-node/src/network/reqresp/handlers/lightClientBootstrap.ts b/packages/beacon-node/src/network/reqresp/handlers/lightClientBootstrap.ts index 06277eb533a3..d14e0945e977 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/lightClientBootstrap.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/lightClientBootstrap.ts @@ -8,8 +8,11 @@ import { import {Root} from "@lodestar/types"; import {IBeaconChain} from "../../../chain/index.js"; import {ReqRespMethod, responseSszTypeByMethod} from "../types.js"; +import {assertLightClientServer} from "../../../node/utils/lightclient.js"; export async function* onLightClientBootstrap(requestBody: Root, chain: IBeaconChain): AsyncIterable { + assertLightClientServer(chain.lightClientServer); + try { const bootstrap = await chain.lightClientServer.getBootstrap(requestBody); const fork = chain.config.getForkName(bootstrap.header.beacon.slot); diff --git a/packages/beacon-node/src/network/reqresp/handlers/lightClientFinalityUpdate.ts b/packages/beacon-node/src/network/reqresp/handlers/lightClientFinalityUpdate.ts index 064f9f6ef4e3..2468b0b64f4b 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/lightClientFinalityUpdate.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/lightClientFinalityUpdate.ts @@ -1,8 +1,11 @@ import {ResponseOutgoing, RespStatus, ResponseError} from "@lodestar/reqresp"; import {IBeaconChain} from "../../../chain/index.js"; import {ReqRespMethod, responseSszTypeByMethod} from "../types.js"; +import {assertLightClientServer} from "../../../node/utils/lightclient.js"; export async function* onLightClientFinalityUpdate(chain: IBeaconChain): AsyncIterable { + assertLightClientServer(chain.lightClientServer); + const update = chain.lightClientServer.getFinalityUpdate(); if (update === null) { throw new ResponseError(RespStatus.RESOURCE_UNAVAILABLE, "No latest finality update available"); diff --git a/packages/beacon-node/src/network/reqresp/handlers/lightClientOptimisticUpdate.ts b/packages/beacon-node/src/network/reqresp/handlers/lightClientOptimisticUpdate.ts index 16001024573e..ba8371910c02 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/lightClientOptimisticUpdate.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/lightClientOptimisticUpdate.ts @@ -1,8 +1,11 @@ import {ResponseOutgoing, ResponseError, RespStatus} from "@lodestar/reqresp"; import {IBeaconChain} from "../../../chain/index.js"; import {ReqRespMethod, responseSszTypeByMethod} from "../types.js"; +import {assertLightClientServer} from "../../../node/utils/lightclient.js"; export async function* onLightClientOptimisticUpdate(chain: IBeaconChain): AsyncIterable { + assertLightClientServer(chain.lightClientServer); + const update = chain.lightClientServer.getOptimisticUpdate(); if (update === null) { throw new ResponseError(RespStatus.RESOURCE_UNAVAILABLE, "No latest optimistic update available"); diff --git a/packages/beacon-node/src/network/reqresp/handlers/lightClientUpdatesByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/lightClientUpdatesByRange.ts index 8dd7e36142cc..eb0e3c3d3f4e 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/lightClientUpdatesByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/lightClientUpdatesByRange.ts @@ -9,11 +9,14 @@ import { } from "@lodestar/reqresp"; import {IBeaconChain} from "../../../chain/index.js"; import {ReqRespMethod, responseSszTypeByMethod} from "../types.js"; +import {assertLightClientServer} from "../../../node/utils/lightclient.js"; export async function* onLightClientUpdatesByRange( requestBody: altair.LightClientUpdatesByRange, chain: IBeaconChain ): AsyncIterable { + assertLightClientServer(chain.lightClientServer); + const count = Math.min(MAX_REQUEST_LIGHT_CLIENT_UPDATES, requestBody.count); for (let period = requestBody.startPeriod; period < requestBody.startPeriod + count; period++) { try { diff --git a/packages/beacon-node/src/node/utils/lightclient.ts b/packages/beacon-node/src/node/utils/lightclient.ts new file mode 100644 index 000000000000..99d7b587e06f --- /dev/null +++ b/packages/beacon-node/src/node/utils/lightclient.ts @@ -0,0 +1,7 @@ +import {LightClientServer} from "../../chain/lightClient/index.js"; + +export function assertLightClientServer(server: LightClientServer | undefined): asserts server is LightClientServer { + if (!server) { + throw Error("Light client server is disabled"); + } +} diff --git a/packages/beacon-node/test/unit/chain/validation/lightClientFinalityUpdate.test.ts b/packages/beacon-node/test/unit/chain/validation/lightClientFinalityUpdate.test.ts index 70dda2535521..ef4364abc366 100644 --- a/packages/beacon-node/test/unit/chain/validation/lightClientFinalityUpdate.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/lightClientFinalityUpdate.test.ts @@ -2,6 +2,7 @@ import {describe, it, expect, beforeEach, afterEach, vi} from "vitest"; import {createChainForkConfig, defaultChainConfig} from "@lodestar/config"; import {altair, ssz} from "@lodestar/types"; import {computeTimeAtSlot} from "@lodestar/state-transition"; +import {RequiredSelective} from "@lodestar/utils"; import {validateLightClientFinalityUpdate} from "../../../../src/chain/validation/lightClientFinalityUpdate.js"; import {LightClientErrorCode} from "../../../../src/chain/errors/lightClientError.js"; import {IBeaconChain} from "../../../../src/chain/index.js"; @@ -30,7 +31,7 @@ describe("Light Client Finality Update validation", function () { } }); - function mockChain(): IBeaconChain { + function mockChain(): RequiredSelective { const chain = getMockedBeaconChain(); vi.spyOn(chain, "genesisTime", "get").mockReturnValue(0); return chain; diff --git a/packages/beacon-node/test/unit/chain/validation/lightClientOptimisticUpdate.test.ts b/packages/beacon-node/test/unit/chain/validation/lightClientOptimisticUpdate.test.ts index 386a08641273..b63a08757380 100644 --- a/packages/beacon-node/test/unit/chain/validation/lightClientOptimisticUpdate.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/lightClientOptimisticUpdate.test.ts @@ -2,6 +2,7 @@ import {describe, it, expect, beforeEach, afterEach, vi} from "vitest"; import {createChainForkConfig, defaultChainConfig} from "@lodestar/config"; import {altair, ssz} from "@lodestar/types"; import {computeTimeAtSlot} from "@lodestar/state-transition"; +import {RequiredSelective} from "@lodestar/utils"; import {validateLightClientOptimisticUpdate} from "../../../../src/chain/validation/lightClientOptimisticUpdate.js"; import {LightClientErrorCode} from "../../../../src/chain/errors/lightClientError.js"; import {IBeaconChain} from "../../../../src/chain/index.js"; @@ -30,7 +31,7 @@ describe("Light Client Optimistic Update validation", function () { } }); - function mockChain(): IBeaconChain { + function mockChain(): RequiredSelective { const chain = getMockedBeaconChain({config}); vi.spyOn(chain, "genesisTime", "get").mockReturnValue(0); vi.spyOn(chain.lightClientServer, "getOptimisticUpdate"); diff --git a/packages/cli/src/cmds/beacon/handler.ts b/packages/cli/src/cmds/beacon/handler.ts index 4d788efa231a..42128b45e4a5 100644 --- a/packages/cli/src/cmds/beacon/handler.ts +++ b/packages/cli/src/cmds/beacon/handler.ts @@ -194,6 +194,10 @@ export async function beaconHandlerInit(args: BeaconArgs & GlobalArgs) { // Inject ENR to beacon options beaconNodeOptions.set({network: {discv5: {enr: enr.encodeTxt(), config: {enrUpdate: !enr.ip && !enr.ip6}}}}); + if (args.disableLightClientServer) { + beaconNodeOptions.set({chain: {disableLightClientServer: true}}); + } + if (args.private) { beaconNodeOptions.set({network: {private: true}}); } else { diff --git a/packages/cli/src/cmds/beacon/options.ts b/packages/cli/src/cmds/beacon/options.ts index 1ec508696a13..5b57b58e82e2 100644 --- a/packages/cli/src/cmds/beacon/options.ts +++ b/packages/cli/src/cmds/beacon/options.ts @@ -22,6 +22,7 @@ type BeaconExtraArgs = { private?: boolean; validatorMonitorLogs?: boolean; attachToGlobalThis?: boolean; + disableLightClientServer?: boolean; }; export const beaconExtraOptions: CliCommandOptions = { @@ -140,6 +141,11 @@ export const beaconExtraOptions: CliCommandOptions = { description: "Attach the beacon node to `globalThis`. Useful to inspect a running beacon node.", type: "boolean", }, + + disableLightClientServer: { + description: "Disable light client server.", + type: "boolean", + }, }; type ENRArgs = { diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 29e2cac2da21..2057e50e07bc 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -16,7 +16,7 @@ export * from "./notNullish.js"; export * from "./sleep.js"; export * from "./sort.js"; export * from "./timeout.js"; -export {type RecursivePartial, bnToNum} from "./types.js"; +export {type RecursivePartial, bnToNum, type RequiredSelective} from "./types.js"; export * from "./url.js"; export * from "./verifyMerkleBranch.js"; export * from "./promise.js"; diff --git a/packages/utils/src/types.ts b/packages/utils/src/types.ts index 935c13cda2c1..62b04ca9a72f 100644 --- a/packages/utils/src/types.ts +++ b/packages/utils/src/types.ts @@ -25,3 +25,10 @@ export type NonEmptyArray = [T, ...T[]]; export type ArrayToTuple> = { [Index in keyof Tuple]: Tuple[Index]; }; + +/** + * Convert optional attributes of an object to required + */ +export type RequiredSelective = T & { + [K in Keys]-?: T[K]; +}; From 5f375804dffdf6d6f275c498454cfcbdda6926c4 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 18 Jul 2024 18:47:03 +0100 Subject: [PATCH 02/11] chore: pin nodejs version to 22.4 (#6964) * chore: pin nodejs version to 22.4 * More pinning --- .github/workflows/benchmark.yml | 2 +- .github/workflows/binaries.yml | 2 +- .github/workflows/docs-check.yml | 4 ++-- .github/workflows/docs.yml | 8 ++++---- .github/workflows/publish-dev.yml | 4 ++-- .github/workflows/publish-rc.yml | 2 +- .github/workflows/publish-stable.yml | 2 +- .github/workflows/test-sim-merge.yml | 2 +- .github/workflows/test-sim.yml | 12 ++++++------ .github/workflows/test.yml | 14 +++++++------- Dockerfile | 6 +++--- package.json | 2 +- 12 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e515bef7f92a..671434fe19ad 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -33,7 +33,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 check-latest: true cache: yarn - name: Node.js version diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index 469b0803c378..722894424b91 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -42,7 +42,7 @@ jobs: sudo apt-get install -y build-essential python3 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - run: | mkdir -p dist yarn global add caxa@3.0.1 diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index 021e5019760d..bd7310995d62 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -15,8 +15,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 - cache: yarn + node-version: 22.4 + cache: yarn - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 63db55435edf..a4c0f18cdbe3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,9 +7,9 @@ on: workflow_dispatch: inputs: ref: - description: 'Ref to deploy, defaults to `unstable`' + description: "Ref to deploy, defaults to `unstable`" required: false - default: 'unstable' + default: "unstable" type: string jobs: @@ -31,9 +31,9 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 check-latest: true - cache: yarn + cache: yarn - name: Node.js version id: node diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index da045764dd41..fb4197b75da9 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -20,10 +20,10 @@ jobs: fetch-depth: 0 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 registry-url: "https://registry.npmjs.org" check-latest: true - cache: yarn + cache: yarn - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index 35831822bcdc..f8f3b21ff349 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -61,7 +61,7 @@ jobs: - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Generate changelog run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 741c060e4d20..9c41693f26f2 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -67,7 +67,7 @@ jobs: - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Generate changelog run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md diff --git a/.github/workflows/test-sim-merge.yml b/.github/workflows/test-sim-merge.yml index ad79bc2c0035..0042a9337bc3 100644 --- a/.github/workflows/test-sim-merge.yml +++ b/.github/workflows/test-sim-merge.yml @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 check-latest: true cache: yarn - name: Node.js version diff --git a/.github/workflows/test-sim.yml b/.github/workflows/test-sim.yml index ff28149537d3..fbe2691da637 100644 --- a/.github/workflows/test-sim.yml +++ b/.github/workflows/test-sim.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 sim-test-multifork: name: Multifork sim test @@ -42,7 +42,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -71,7 +71,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -100,7 +100,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -129,7 +129,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -158,7 +158,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f55eb661629f..c801b0462d60 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 @@ -95,7 +95,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" @@ -134,7 +134,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -171,7 +171,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -195,7 +195,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" diff --git a/Dockerfile b/Dockerfile index fdab1140af47..c65cac28d51d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # --platform=$BUILDPLATFORM is used build javascript source with host arch # Otherwise TS builds on emulated archs and can be extremely slow (+1h) -FROM --platform=${BUILDPLATFORM:-amd64} node:22-alpine as build_src +FROM --platform=${BUILDPLATFORM:-amd64} node:22.4-alpine as build_src ARG COMMIT WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 py3-setuptools && rm -rf /var/cache/apk/* @@ -21,7 +21,7 @@ RUN cd packages/cli && GIT_COMMIT=${COMMIT} yarn write-git-data # Copy built src + node_modules to build native packages for archs different than host. # Note: This step is redundant for the host arch -FROM node:22-alpine as build_deps +FROM node:22.4-alpine as build_deps WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 py3-setuptools && rm -rf /var/cache/apk/* @@ -35,7 +35,7 @@ RUN cd node_modules/classic-level && yarn rebuild # Copy built src + node_modules to a new layer to prune unnecessary fs # Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020) -FROM node:22-alpine +FROM node:22.4-alpine WORKDIR /usr/app COPY --from=build_deps /usr/app . diff --git a/package.json b/package.json index e4464b3b5a9f..87ccd7e9f031 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "root", "private": true, "engines": { - "node": ">=20.1.0 <21 || >=22 <23" + "node": ">=20.1.0 <21 || >=22 <22.5" }, "packageManager": "yarn@1.22.22+sha256.c17d3797fb9a9115bf375e31bfd30058cac6bc9c3b8807a3d8cb2094794b51ca", "workspaces": [ From fed08fe5107a809d8862494ae6a86ebc6f2f0ee9 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 18 Jul 2024 14:07:10 -0400 Subject: [PATCH 03/11] chore: refactor block and state api utils (#6963) * chore: refactor block and state api utils * Update packages/beacon-node/src/api/impl/beacon/blocks/utils.ts Co-authored-by: Nico Flaig * Update packages/beacon-node/src/api/impl/beacon/state/utils.ts Co-authored-by: Nico Flaig * chore: address pr comments * chore: tiny cleanup * Use more informative return types * Align block and state utils * Let server handler deserialize state bytes --------- Co-authored-by: Nico Flaig --- .../src/api/impl/beacon/blocks/index.ts | 14 ++-- .../src/api/impl/beacon/blocks/utils.ts | 50 ++++++------ .../src/api/impl/beacon/rewards/index.ts | 6 +- .../src/api/impl/beacon/state/index.ts | 14 ++-- .../src/api/impl/beacon/state/utils.ts | 80 +++++++++++-------- .../beacon-node/src/api/impl/debug/index.ts | 18 ++++- .../beacon-node/src/api/impl/proof/index.ts | 8 +- 7 files changed, 110 insertions(+), 80 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 177f58aebb95..1b8a59cc8967 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -34,7 +34,7 @@ import {ApiModules} from "../../types.js"; import {validateGossipBlock} from "../../../../chain/validation/block.js"; import {verifyBlocksInEpoch} from "../../../../chain/blocks/verifyBlock.js"; import {BeaconChain} from "../../../../chain/chain.js"; -import {resolveBlockId, toBeaconHeaderResponse} from "./utils.js"; +import {getBlockResponse, toBeaconHeaderResponse} from "./utils.js"; type PublishBlockOpts = ImportBlockOpts; @@ -371,7 +371,7 @@ export function getBeaconBlockApi({ }, async getBlockHeader({blockId}) { - const {block, executionOptimistic, finalized} = await resolveBlockId(chain, blockId); + const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); return { data: toBeaconHeaderResponse(config, block, true), meta: {executionOptimistic, finalized}, @@ -379,7 +379,7 @@ export function getBeaconBlockApi({ }, async getBlockV2({blockId}) { - const {block, executionOptimistic, finalized} = await resolveBlockId(chain, blockId); + const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); return { data: block, meta: { @@ -391,7 +391,7 @@ export function getBeaconBlockApi({ }, async getBlindedBlock({blockId}) { - const {block, executionOptimistic, finalized} = await resolveBlockId(chain, blockId); + const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); const fork = config.getForkName(block.message.slot); return { data: isForkExecution(fork) @@ -406,7 +406,7 @@ export function getBeaconBlockApi({ }, async getBlockAttestations({blockId}) { - const {block, executionOptimistic, finalized} = await resolveBlockId(chain, blockId); + const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); return { data: Array.from(block.message.body.attestations), meta: {executionOptimistic, finalized}, @@ -445,7 +445,7 @@ export function getBeaconBlockApi({ } // Slow path - const {block, executionOptimistic, finalized} = await resolveBlockId(chain, blockId); + const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); return { data: {root: config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message)}, meta: {executionOptimistic, finalized}, @@ -464,7 +464,7 @@ export function getBeaconBlockApi({ }, async getBlobSidecars({blockId, indices}) { - const {block, executionOptimistic, finalized} = await resolveBlockId(chain, blockId); + const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); let {blobSidecars} = (await db.blobSidecars.get(blockRoot)) ?? {}; diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/utils.ts b/packages/beacon-node/src/api/impl/beacon/blocks/utils.ts index f0d243967c22..fe4fc5ca3dc0 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/utils.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/utils.ts @@ -1,7 +1,8 @@ import {routes} from "@lodestar/api"; import {blockToHeader} from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; -import {SignedBeaconBlock} from "@lodestar/types"; +import {RootHex, SignedBeaconBlock, Slot} from "@lodestar/types"; +import {IForkChoice} from "@lodestar/fork-choice"; import {GENESIS_SLOT} from "../../../../constants/index.js"; import {ApiError, ValidationError} from "../../errors.js"; import {IBeaconChain} from "../../../../chain/interface.js"; @@ -22,44 +23,29 @@ export function toBeaconHeaderResponse( }; } -export async function resolveBlockId( - chain: IBeaconChain, - blockId: routes.beacon.BlockId -): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean}> { - const res = await resolveBlockIdOrNull(chain, blockId); - if (!res) { - throw new ApiError(404, `No block found for id '${blockId}'`); - } - - return res; -} - -async function resolveBlockIdOrNull( - chain: IBeaconChain, - blockId: routes.beacon.BlockId -): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null> { +export function resolveBlockId(forkChoice: IForkChoice, blockId: routes.beacon.BlockId): RootHex | Slot { blockId = String(blockId).toLowerCase(); if (blockId === "head") { - return chain.getBlockByRoot(chain.forkChoice.getHead().blockRoot); + return forkChoice.getHead().blockRoot; } if (blockId === "genesis") { - return chain.getCanonicalBlockAtSlot(GENESIS_SLOT); + return GENESIS_SLOT; } if (blockId === "finalized") { - return chain.getCanonicalBlockAtSlot(chain.forkChoice.getFinalizedBlock().slot); + return forkChoice.getFinalizedBlock().blockRoot; } if (blockId === "justified") { - return chain.getBlockByRoot(chain.forkChoice.getJustifiedBlock().blockRoot); + return forkChoice.getJustifiedBlock().blockRoot; } if (blockId.startsWith("0x")) { if (!rootHexRegex.test(blockId)) { throw new ValidationError(`Invalid block id '${blockId}'`, "blockId"); } - return chain.getBlockByRoot(blockId); + return blockId; } // block id must be slot @@ -67,5 +53,23 @@ async function resolveBlockIdOrNull( if (isNaN(blockSlot) && isNaN(blockSlot - 0)) { throw new ValidationError(`Invalid block id '${blockId}'`, "blockId"); } - return chain.getCanonicalBlockAtSlot(blockSlot); + return blockSlot; +} + +export async function getBlockResponse( + chain: IBeaconChain, + blockId: routes.beacon.BlockId +): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean}> { + const rootOrSlot = resolveBlockId(chain.forkChoice, blockId); + + const res = + typeof rootOrSlot === "string" + ? await chain.getBlockByRoot(rootOrSlot) + : await chain.getCanonicalBlockAtSlot(rootOrSlot); + + if (!res) { + throw new ApiError(404, `No block found for id '${blockId}'`); + } + + return res; } diff --git a/packages/beacon-node/src/api/impl/beacon/rewards/index.ts b/packages/beacon-node/src/api/impl/beacon/rewards/index.ts index 96399db27b4f..8d8a77701b91 100644 --- a/packages/beacon-node/src/api/impl/beacon/rewards/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/rewards/index.ts @@ -1,14 +1,14 @@ import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; import {ApiModules} from "../../types.js"; -import {resolveBlockId} from "../blocks/utils.js"; +import {getBlockResponse} from "../blocks/utils.js"; export function getBeaconRewardsApi({ chain, }: Pick): ApplicationMethods { return { async getBlockRewards({blockId}) { - const {block, executionOptimistic, finalized} = await resolveBlockId(chain, blockId); + const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); const data = await chain.getBlockRewards(block.message); return {data, meta: {executionOptimistic, finalized}}; }, @@ -17,7 +17,7 @@ export function getBeaconRewardsApi({ return {data: rewards, meta: {executionOptimistic, finalized}}; }, async getSyncCommitteeRewards({blockId, validatorIds}) { - const {block, executionOptimistic, finalized} = await resolveBlockId(chain, blockId); + const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); const data = await chain.getSyncCommitteeRewards(block.message, validatorIds); return {data, meta: {executionOptimistic, finalized}}; }, diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index 201cc875123e..9d9646ee8cf3 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -15,7 +15,7 @@ import { filterStateValidatorsByStatus, getStateValidatorIndex, getValidatorStatus, - resolveStateId, + getStateResponse, toValidatorResponse, } from "./utils.js"; @@ -26,7 +26,7 @@ export function getBeaconStateApi({ async function getState( stateId: routes.beacon.StateId ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean}> { - return resolveStateId(chain, stateId); + return getStateResponse(chain, stateId); } return { @@ -76,7 +76,7 @@ export function getBeaconStateApi({ }, async getStateValidators({stateId, validatorIds = [], statuses = []}) { - const {state, executionOptimistic, finalized} = await resolveStateId(chain, stateId); + const {state, executionOptimistic, finalized} = await getStateResponse(chain, stateId); const currentEpoch = getCurrentEpoch(state); const {validators, balances} = state; // Get the validators sub tree once for all the loop const {pubkey2index} = chain.getHeadState().epochCtx; @@ -131,7 +131,7 @@ export function getBeaconStateApi({ }, async getStateValidator({stateId, validatorId}) { - const {state, executionOptimistic, finalized} = await resolveStateId(chain, stateId); + const {state, executionOptimistic, finalized} = await getStateResponse(chain, stateId); const {pubkey2index} = chain.getHeadState().epochCtx; const resp = getStateValidatorIndex(validatorId, state, pubkey2index); @@ -152,7 +152,7 @@ export function getBeaconStateApi({ }, async getStateValidatorBalances({stateId, validatorIds = []}) { - const {state, executionOptimistic, finalized} = await resolveStateId(chain, stateId); + const {state, executionOptimistic, finalized} = await getStateResponse(chain, stateId); if (validatorIds.length) { const headState = chain.getHeadState(); @@ -193,7 +193,7 @@ export function getBeaconStateApi({ }, async getEpochCommittees({stateId, ...filters}) { - const {state, executionOptimistic, finalized} = await resolveStateId(chain, stateId); + const {state, executionOptimistic, finalized} = await getStateResponse(chain, stateId); const stateCached = state as CachedBeaconStateAltair; if (stateCached.epochCtx === undefined) { @@ -235,7 +235,7 @@ export function getBeaconStateApi({ */ async getEpochSyncCommittees({stateId, epoch}) { // TODO: Should pick a state with the provided epoch too - const {state, executionOptimistic, finalized} = await resolveStateId(chain, stateId); + const {state, executionOptimistic, finalized} = await getStateResponse(chain, stateId); // TODO: If possible compute the syncCommittees in advance of the fork and expose them here. // So the validators can prepare and potentially attest the first block. Not critical tho, it's very unlikely diff --git a/packages/beacon-node/src/api/impl/beacon/state/utils.ts b/packages/beacon-node/src/api/impl/beacon/state/utils.ts index 73f7134e1530..a2079afaf9b8 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/utils.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/utils.ts @@ -1,53 +1,31 @@ import {routes} from "@lodestar/api"; import {FAR_FUTURE_EPOCH, GENESIS_SLOT} from "@lodestar/params"; import {BeaconStateAllForks, PubkeyIndexMap} from "@lodestar/state-transition"; -import {BLSPubkey, Epoch, phase0, ValidatorIndex} from "@lodestar/types"; +import {BLSPubkey, Epoch, phase0, RootHex, Slot, ValidatorIndex} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; -import {IBeaconChain, StateGetOpts} from "../../../../chain/index.js"; +import {IForkChoice} from "@lodestar/fork-choice"; +import {IBeaconChain} from "../../../../chain/index.js"; import {ApiError, ValidationError} from "../../errors.js"; -import {isOptimisticBlock} from "../../../../util/forkChoice.js"; -export async function resolveStateId( - chain: IBeaconChain, - stateId: routes.beacon.StateId, - opts?: StateGetOpts -): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean}> { - const stateRes = await resolveStateIdOrNull(chain, stateId, opts); - if (!stateRes) { - throw new ApiError(404, `No state found for id '${stateId}'`); - } - - return stateRes; -} - -async function resolveStateIdOrNull( - chain: IBeaconChain, - stateId: routes.beacon.StateId, - opts?: StateGetOpts -): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null> { +export function resolveStateId(forkChoice: IForkChoice, stateId: routes.beacon.StateId): RootHex | Slot { if (stateId === "head") { - // TODO: This is not OK, head and headState must be fetched atomically - const head = chain.forkChoice.getHead(); - const headState = chain.getHeadState(); - return {state: headState, executionOptimistic: isOptimisticBlock(head), finalized: false}; + return forkChoice.getHead().stateRoot; } if (stateId === "genesis") { - return chain.getStateBySlot(GENESIS_SLOT, opts); + return GENESIS_SLOT; } if (stateId === "finalized") { - const checkpoint = chain.forkChoice.getFinalizedCheckpoint(); - return chain.getStateByCheckpoint(checkpoint); + return forkChoice.getFinalizedBlock().stateRoot; } if (stateId === "justified") { - const checkpoint = chain.forkChoice.getJustifiedCheckpoint(); - return chain.getStateByCheckpoint(checkpoint); + return forkChoice.getJustifiedBlock().stateRoot; } if (typeof stateId === "string" && stateId.startsWith("0x")) { - return chain.getStateByStateRoot(stateId, opts); + return stateId; } // id must be slot @@ -56,7 +34,45 @@ async function resolveStateIdOrNull( throw new ValidationError(`Invalid block id '${stateId}'`, "blockId"); } - return chain.getStateBySlot(blockSlot, opts); + return blockSlot; +} + +export async function getStateResponse( + chain: IBeaconChain, + stateId: routes.beacon.StateId +): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean}> { + const rootOrSlot = resolveStateId(chain.forkChoice, stateId); + + const res = + typeof rootOrSlot === "string" + ? await chain.getStateByStateRoot(rootOrSlot) + : await chain.getStateBySlot(rootOrSlot); + + if (!res) { + throw new ApiError(404, `No state found for id '${stateId}'`); + } + + return res; +} + +export async function getStateResponseWithRegen( + chain: IBeaconChain, + stateId: routes.beacon.StateId +): Promise<{state: BeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean}> { + const rootOrSlot = resolveStateId(chain.forkChoice, stateId); + + const res = + typeof rootOrSlot === "string" + ? await chain.getStateByStateRoot(rootOrSlot, {allowRegen: true}) + : rootOrSlot >= chain.forkChoice.getFinalizedBlock().slot + ? await chain.getStateBySlot(rootOrSlot, {allowRegen: true}) + : null; // TODO implement historical state regen + + if (!res) { + throw new ApiError(404, `No state found for id '${stateId}'`); + } + + return res; } /** diff --git a/packages/beacon-node/src/api/impl/debug/index.ts b/packages/beacon-node/src/api/impl/debug/index.ts index f1254ae1b7fb..4edb8ba9b2dd 100644 --- a/packages/beacon-node/src/api/impl/debug/index.ts +++ b/packages/beacon-node/src/api/impl/debug/index.ts @@ -1,8 +1,10 @@ import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; -import {resolveStateId} from "../beacon/state/utils.js"; +import {BeaconState} from "@lodestar/types"; +import {getStateResponseWithRegen} from "../beacon/state/utils.js"; import {ApiModules} from "../types.js"; import {isOptimisticBlock} from "../../../util/forkChoice.js"; +import {getStateSlotFromBytes} from "../../../util/multifork.js"; export function getDebugApi({ chain, @@ -34,11 +36,19 @@ export function getDebugApi({ }, async getStateV2({stateId}, context) { - const {state, executionOptimistic, finalized} = await resolveStateId(chain, stateId, {allowRegen: true}); + const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, stateId); + let slot: number, data: Uint8Array | BeaconState; + if (state instanceof Uint8Array) { + slot = getStateSlotFromBytes(state); + data = state; + } else { + slot = state.slot; + data = context?.returnBytes ? state.serialize() : state.toValue(); + } return { - data: context?.returnBytes ? state.serialize() : state.toValue(), + data, meta: { - version: config.getForkName(state.slot), + version: config.getForkName(slot), executionOptimistic, finalized, }, diff --git a/packages/beacon-node/src/api/impl/proof/index.ts b/packages/beacon-node/src/api/impl/proof/index.ts index a581fb8eed59..9e1a33940225 100644 --- a/packages/beacon-node/src/api/impl/proof/index.ts +++ b/packages/beacon-node/src/api/impl/proof/index.ts @@ -2,8 +2,8 @@ import {CompactMultiProof, createProof, ProofType} from "@chainsafe/persistent-m import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; import {ApiModules} from "../types.js"; -import {resolveStateId} from "../beacon/state/utils.js"; -import {resolveBlockId} from "../beacon/blocks/utils.js"; +import {getStateResponse} from "../beacon/state/utils.js"; +import {getBlockResponse} from "../beacon/blocks/utils.js"; import {ApiOptions} from "../../options.js"; export function getProofApi( @@ -21,7 +21,7 @@ export function getProofApi( throw new Error("Requested proof is too large."); } - const {state} = await resolveStateId(chain, stateId); + const {state} = await getStateResponse(chain, stateId); // Commit any changes before computing the state root. In normal cases the state should have no changes here state.commit(); @@ -40,7 +40,7 @@ export function getProofApi( throw new Error("Requested proof is too large."); } - const {block} = await resolveBlockId(chain, blockId); + const {block} = await getBlockResponse(chain, blockId); // Commit any changes before computing the state root. In normal cases the state should have no changes here const blockNode = config.getForkTypes(block.message.slot).BeaconBlock.toView(block.message).node; From 3fb0f5cc6c121e154fef3ee06c0ffd6f59a4c840 Mon Sep 17 00:00:00 2001 From: g11tech Date: Fri, 19 Jul 2024 14:54:05 +0530 Subject: [PATCH 04/11] fix: fix the publish blinded block api parsing for optional header verison (#6966) --- packages/api/src/beacon/routes/beacon/block.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts index cbf206e51a4a..73680ac0afc2 100644 --- a/packages/api/src/beacon/routes/beacon/block.ts +++ b/packages/api/src/beacon/routes/beacon/block.ts @@ -444,7 +444,16 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { - const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); + let fork: ForkName; + // As per spec, version header is optional for JSON requests + const versionHeader = fromHeaders(headers, MetaHeader.Version, false); + if (versionHeader !== undefined) { + fork = toForkName(versionHeader); + } else { + // Determine fork from slot in JSON payload + fork = config.getForkName((body as SignedBlindedBeaconBlock).message.slot); + } + return { signedBlindedBlock: getExecutionForkTypes(fork).SignedBlindedBeaconBlock.fromJson(body), }; From 01fde1283fcc6b2a0a129e848528ec9d7443579d Mon Sep 17 00:00:00 2001 From: Phil Ngo Date: Fri, 19 Jul 2024 07:55:30 -0400 Subject: [PATCH 05/11] chore: bump package versions to 1.20.2 --- lerna.json | 2 +- packages/api/package.json | 10 +++++----- packages/beacon-node/package.json | 26 +++++++++++++------------- packages/cli/package.json | 26 +++++++++++++------------- packages/config/package.json | 6 +++--- packages/db/package.json | 8 ++++---- packages/flare/package.json | 14 +++++++------- packages/fork-choice/package.json | 12 ++++++------ packages/light-client/package.json | 12 ++++++------ packages/logger/package.json | 6 +++--- packages/params/package.json | 2 +- packages/prover/package.json | 18 +++++++++--------- packages/reqresp/package.json | 12 ++++++------ packages/spec-test-util/package.json | 4 ++-- packages/state-transition/package.json | 10 +++++----- packages/test-utils/package.json | 6 +++--- packages/types/package.json | 4 ++-- packages/utils/package.json | 2 +- packages/validator/package.json | 18 +++++++++--------- 19 files changed, 99 insertions(+), 99 deletions(-) diff --git a/lerna.json b/lerna.json index bfba68d36c54..ccdcaca872ec 100644 --- a/lerna.json +++ b/lerna.json @@ -4,7 +4,7 @@ ], "npmClient": "yarn", "useNx": true, - "version": "1.20.1", + "version": "1.20.2", "stream": true, "command": { "version": { diff --git a/packages/api/package.json b/packages/api/package.json index 280d4bf4e9ef..5b91b46c128e 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -72,10 +72,10 @@ "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "eventsource": "^2.0.2", "qs": "^6.11.1" }, diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index c57de6eb5781..80d61cb119cb 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -120,18 +120,18 @@ "@libp2p/peer-id-factory": "^4.1.0", "@libp2p/prometheus-metrics": "^3.0.21", "@libp2p/tcp": "9.0.23", - "@lodestar/api": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/db": "^1.20.1", - "@lodestar/fork-choice": "^1.20.1", - "@lodestar/light-client": "^1.20.1", - "@lodestar/logger": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/reqresp": "^1.20.1", - "@lodestar/state-transition": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", - "@lodestar/validator": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/db": "^1.20.2", + "@lodestar/fork-choice": "^1.20.2", + "@lodestar/light-client": "^1.20.2", + "@lodestar/logger": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/reqresp": "^1.20.2", + "@lodestar/state-transition": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", + "@lodestar/validator": "^1.20.2", "@multiformats/multiaddr": "^12.1.3", "c-kzg": "^2.1.2", "datastore-core": "^9.1.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 4f27d030c735..30b446c9fd1b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.20.1", + "version": "1.20.2", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -63,17 +63,17 @@ "@libp2p/crypto": "^4.1.0", "@libp2p/peer-id": "^4.1.0", "@libp2p/peer-id-factory": "^4.1.0", - "@lodestar/api": "^1.20.1", - "@lodestar/beacon-node": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/db": "^1.20.1", - "@lodestar/light-client": "^1.20.1", - "@lodestar/logger": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/state-transition": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", - "@lodestar/validator": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/beacon-node": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/db": "^1.20.2", + "@lodestar/light-client": "^1.20.2", + "@lodestar/logger": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/state-transition": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", + "@lodestar/validator": "^1.20.2", "@multiformats/multiaddr": "^12.1.3", "deepmerge": "^4.3.1", "ethers": "^6.7.0", @@ -89,7 +89,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.20.1", + "@lodestar/test-utils": "^1.20.2", "@types/debug": "^4.1.7", "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", diff --git a/packages/config/package.json b/packages/config/package.json index f25d67bb687d..8d5fd3d80c35 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.20.1", + "version": "1.20.2", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -65,7 +65,7 @@ ], "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/params": "^1.20.1", - "@lodestar/types": "^1.20.1" + "@lodestar/params": "^1.20.2", + "@lodestar/types": "^1.20.2" } } diff --git a/packages/db/package.json b/packages/db/package.json index f883ff39291d..6cfaf6ecce70 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.20.1", + "version": "1.20.2", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", @@ -36,12 +36,12 @@ }, "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/config": "^1.20.2", + "@lodestar/utils": "^1.20.2", "classic-level": "^1.4.1", "it-all": "^3.0.4" }, "devDependencies": { - "@lodestar/logger": "^1.20.1" + "@lodestar/logger": "^1.20.2" } } diff --git a/packages/flare/package.json b/packages/flare/package.json index b9a02da63383..c735642ad8e4 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.20.1", + "version": "1.20.2", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -60,12 +60,12 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/bls-keygen": "^0.4.0", - "@lodestar/api": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/state-transition": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/state-transition": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "source-map-support": "^0.5.21", "yargs": "^17.7.1" }, diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 8352c751aa37..2da624af4581 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": "./lib/index.js", "types": "./lib/index.d.ts", @@ -37,11 +37,11 @@ }, "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/state-transition": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1" + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/state-transition": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 68e8bf6a26dd..8b433375f07f 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -76,11 +76,11 @@ "@chainsafe/bls": "7.1.3", "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/ssz": "^0.15.1", - "@lodestar/api": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "mitt": "^3.0.0" }, "devDependencies": { diff --git a/packages/logger/package.json b/packages/logger/package.json index 15e33b579ba0..117040580d60 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -66,14 +66,14 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@lodestar/utils": "^1.20.1", + "@lodestar/utils": "^1.20.2", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1", "winston-transport": "^4.5.0" }, "devDependencies": { "@chainsafe/threads": "^1.11.1", - "@lodestar/test-utils": "^1.20.1", + "@lodestar/test-utils": "^1.20.2", "@types/triple-beam": "^1.3.2", "triple-beam": "^1.3.0" }, diff --git a/packages/params/package.json b/packages/params/package.json index c8505204c42e..20f50f625cfd 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.20.1", + "version": "1.20.2", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/prover/package.json b/packages/prover/package.json index 577e3bc105e2..f5d8c26105d4 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -69,13 +69,13 @@ "@ethereumjs/tx": "^4.1.2", "@ethereumjs/util": "^8.0.6", "@ethereumjs/vm": "^6.4.2", - "@lodestar/api": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/light-client": "^1.20.1", - "@lodestar/logger": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/light-client": "^1.20.2", + "@lodestar/logger": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "ethereum-cryptography": "^2.0.0", "find-up": "^6.3.0", "http-proxy": "^1.18.1", @@ -84,7 +84,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.20.1", + "@lodestar/test-utils": "^1.20.2", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 8091e4f16ee7..37c16318c5b3 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -54,9 +54,9 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.1.1", "@libp2p/interface": "^1.3.0", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/utils": "^1.20.2", "it-all": "^3.0.4", "it-pipe": "^3.0.1", "snappy": "^7.2.2", @@ -65,8 +65,8 @@ "uint8arraylist": "^2.4.7" }, "devDependencies": { - "@lodestar/logger": "^1.20.1", - "@lodestar/types": "^1.20.1", + "@lodestar/logger": "^1.20.2", + "@lodestar/types": "^1.20.2", "libp2p": "1.4.3" }, "peerDependencies": { diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index f5f590d55fa0..bb2fe357c5f8 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.20.1", + "version": "1.20.2", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -62,7 +62,7 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.20.1", + "@lodestar/utils": "^1.20.2", "axios": "^1.3.4", "rimraf": "^4.4.1", "snappyjs": "^0.7.0", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 7dc542762cbf..b032f5204af3 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -64,10 +64,10 @@ "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "bigint-buffer": "^1.1.5" }, "keywords": [ diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 45fb3d0e0909..d2571a974b65 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.20.1", + "version": "1.20.2", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -59,8 +59,8 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/bls-keystore": "^3.0.1", - "@lodestar/params": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/params": "^1.20.2", + "@lodestar/utils": "^1.20.2", "axios": "^1.3.4", "testcontainers": "^10.2.1", "tmp": "^0.2.1", diff --git a/packages/types/package.json b/packages/types/package.json index 6d69d501134f..d5512763b197 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -71,7 +71,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/params": "^1.20.1", + "@lodestar/params": "^1.20.2", "ethereum-cryptography": "^2.0.0" }, "keywords": [ diff --git a/packages/utils/package.json b/packages/utils/package.json index 8ad6f8b6b93b..e3ada3e2901b 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": "./lib/index.js", "files": [ diff --git a/packages/validator/package.json b/packages/validator/package.json index c4929ffb6eca..5290698d4adb 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.20.1", + "version": "1.20.2", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -47,17 +47,17 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/ssz": "^0.15.1", - "@lodestar/api": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/db": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/state-transition": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/db": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/state-transition": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { - "@lodestar/test-utils": "^1.20.1", + "@lodestar/test-utils": "^1.20.2", "bigint-buffer": "^1.1.5", "rimraf": "^4.4.1" } From fb6bef857bb8f0535f150c29d6c9a9e1faa29999 Mon Sep 17 00:00:00 2001 From: g11tech Date: Fri, 19 Jul 2024 14:54:05 +0530 Subject: [PATCH 06/11] fix: fix the publish blinded block api parsing for optional header verison (#6966) --- packages/api/src/beacon/routes/beacon/block.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts index cbf206e51a4a..73680ac0afc2 100644 --- a/packages/api/src/beacon/routes/beacon/block.ts +++ b/packages/api/src/beacon/routes/beacon/block.ts @@ -444,7 +444,16 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { - const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); + let fork: ForkName; + // As per spec, version header is optional for JSON requests + const versionHeader = fromHeaders(headers, MetaHeader.Version, false); + if (versionHeader !== undefined) { + fork = toForkName(versionHeader); + } else { + // Determine fork from slot in JSON payload + fork = config.getForkName((body as SignedBlindedBeaconBlock).message.slot); + } + return { signedBlindedBlock: getExecutionForkTypes(fork).SignedBlindedBeaconBlock.fromJson(body), }; From f20484bb4b6b5f3a27c624556d1e6fb9f448b969 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 18 Jul 2024 18:47:03 +0100 Subject: [PATCH 07/11] chore: pin nodejs version to 22.4 (#6964) * chore: pin nodejs version to 22.4 * More pinning --- .github/workflows/benchmark.yml | 2 +- .github/workflows/binaries.yml | 2 +- .github/workflows/docs-check.yml | 4 ++-- .github/workflows/docs.yml | 8 ++++---- .github/workflows/publish-dev.yml | 4 ++-- .github/workflows/publish-rc.yml | 2 +- .github/workflows/publish-stable.yml | 2 +- .github/workflows/test-sim-merge.yml | 2 +- .github/workflows/test-sim.yml | 12 ++++++------ .github/workflows/test.yml | 14 +++++++------- Dockerfile | 6 +++--- package.json | 2 +- 12 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e515bef7f92a..671434fe19ad 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -33,7 +33,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 check-latest: true cache: yarn - name: Node.js version diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index 469b0803c378..722894424b91 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -42,7 +42,7 @@ jobs: sudo apt-get install -y build-essential python3 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - run: | mkdir -p dist yarn global add caxa@3.0.1 diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index 021e5019760d..bd7310995d62 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -15,8 +15,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 - cache: yarn + node-version: 22.4 + cache: yarn - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 63db55435edf..a4c0f18cdbe3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,9 +7,9 @@ on: workflow_dispatch: inputs: ref: - description: 'Ref to deploy, defaults to `unstable`' + description: "Ref to deploy, defaults to `unstable`" required: false - default: 'unstable' + default: "unstable" type: string jobs: @@ -31,9 +31,9 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 check-latest: true - cache: yarn + cache: yarn - name: Node.js version id: node diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index da045764dd41..fb4197b75da9 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -20,10 +20,10 @@ jobs: fetch-depth: 0 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 registry-url: "https://registry.npmjs.org" check-latest: true - cache: yarn + cache: yarn - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index 35831822bcdc..f8f3b21ff349 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -61,7 +61,7 @@ jobs: - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Generate changelog run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 741c060e4d20..9c41693f26f2 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -67,7 +67,7 @@ jobs: - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Generate changelog run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md diff --git a/.github/workflows/test-sim-merge.yml b/.github/workflows/test-sim-merge.yml index ad79bc2c0035..0042a9337bc3 100644 --- a/.github/workflows/test-sim-merge.yml +++ b/.github/workflows/test-sim-merge.yml @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 check-latest: true cache: yarn - name: Node.js version diff --git a/.github/workflows/test-sim.yml b/.github/workflows/test-sim.yml index ff28149537d3..fbe2691da637 100644 --- a/.github/workflows/test-sim.yml +++ b/.github/workflows/test-sim.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 sim-test-multifork: name: Multifork sim test @@ -42,7 +42,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -71,7 +71,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -100,7 +100,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -129,7 +129,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -158,7 +158,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f55eb661629f..c801b0462d60 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 @@ -95,7 +95,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" @@ -134,7 +134,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -171,7 +171,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -195,7 +195,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" diff --git a/Dockerfile b/Dockerfile index fdab1140af47..c65cac28d51d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # --platform=$BUILDPLATFORM is used build javascript source with host arch # Otherwise TS builds on emulated archs and can be extremely slow (+1h) -FROM --platform=${BUILDPLATFORM:-amd64} node:22-alpine as build_src +FROM --platform=${BUILDPLATFORM:-amd64} node:22.4-alpine as build_src ARG COMMIT WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 py3-setuptools && rm -rf /var/cache/apk/* @@ -21,7 +21,7 @@ RUN cd packages/cli && GIT_COMMIT=${COMMIT} yarn write-git-data # Copy built src + node_modules to build native packages for archs different than host. # Note: This step is redundant for the host arch -FROM node:22-alpine as build_deps +FROM node:22.4-alpine as build_deps WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 py3-setuptools && rm -rf /var/cache/apk/* @@ -35,7 +35,7 @@ RUN cd node_modules/classic-level && yarn rebuild # Copy built src + node_modules to a new layer to prune unnecessary fs # Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020) -FROM node:22-alpine +FROM node:22.4-alpine WORKDIR /usr/app COPY --from=build_deps /usr/app . diff --git a/package.json b/package.json index e4464b3b5a9f..87ccd7e9f031 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "root", "private": true, "engines": { - "node": ">=20.1.0 <21 || >=22 <23" + "node": ">=20.1.0 <21 || >=22 <22.5" }, "packageManager": "yarn@1.22.22+sha256.c17d3797fb9a9115bf375e31bfd30058cac6bc9c3b8807a3d8cb2094794b51ca", "workspaces": [ From 810f4a5a66311bee5c62a4e5eabed37456d15591 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 15 Jul 2024 15:19:58 +0100 Subject: [PATCH 08/11] feat: add option to disable thread pool for keystore decryption (#6949) --- .../keymanager/decryptKeystoreDefinitions.ts | 69 ++++++++++++++----- packages/cli/src/cmds/validator/options.ts | 8 +++ .../cli/src/cmds/validator/signers/index.ts | 2 + .../decryptKeystoreDefinitions.test.ts | 23 ++++--- 4 files changed, 77 insertions(+), 25 deletions(-) diff --git a/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts b/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts index 2901fd6cdfb5..076da70adcdb 100644 --- a/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts +++ b/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts @@ -1,5 +1,7 @@ +import fs from "node:fs"; import path from "node:path"; import bls from "@chainsafe/bls"; +import {Keystore} from "@chainsafe/bls-keystore"; import {SignerLocal, SignerType} from "@lodestar/validator"; import {LogLevel, Logger} from "@lodestar/utils"; import {lockFilepath, unlockFilepath} from "../../../util/lockfile.js"; @@ -7,11 +9,13 @@ import {LocalKeystoreDefinition} from "./interface.js"; import {clearKeystoreCache, loadKeystoreCache, writeKeystoreCache} from "./keystoreCache.js"; import {DecryptKeystoresThreadPool} from "./decryptKeystores/index.js"; -type KeystoreDecryptOptions = { +export type KeystoreDecryptOptions = { ignoreLockFile?: boolean; onDecrypt?: (index: number) => void; // Try to use the cache file if it exists cacheFilePath?: string; + /** Use main thread to decrypt keystores */ + disableThreadPool?: boolean; logger: Pick; signal: AbortSignal; }; @@ -57,14 +61,50 @@ export async function decryptKeystoreDefinitions( const signers = new Array(keystoreCount); const passwords = new Array(keystoreCount); const errors: KeystoreDecryptError[] = []; - const decryptKeystores = new DecryptKeystoresThreadPool(keystoreCount, opts.signal); - for (const [index, definition] of keystoreDefinitions.entries()) { - lockKeystore(definition.keystorePath, opts); + if (!opts.disableThreadPool) { + const decryptKeystores = new DecryptKeystoresThreadPool(keystoreCount, opts.signal); + + for (const [index, definition] of keystoreDefinitions.entries()) { + lockKeystore(definition.keystorePath, opts); + + decryptKeystores.queue( + definition, + (secretKeyBytes: Uint8Array) => { + const signer: SignerLocal = { + type: SignerType.Local, + secretKey: bls.SecretKey.fromBytes(secretKeyBytes), + }; + + signers[index] = signer; + passwords[index] = definition.password; + + if (opts?.onDecrypt) { + opts?.onDecrypt(index); + } + }, + (error: Error) => { + // In-progress tasks can't be canceled, so there's a chance that multiple errors may be caught + // add to the list of errors + errors.push({keystoreFile: path.basename(definition.keystorePath), error}); + // cancel all pending tasks, no need to continue decrypting after we hit one error + decryptKeystores.cancel(); + } + ); + } + + await decryptKeystores.completed(); + } else { + // Decrypt keystores in main thread + for (const [index, definition] of keystoreDefinitions.entries()) { + lockKeystore(definition.keystorePath, opts); + + try { + const keystore = Keystore.parse(fs.readFileSync(definition.keystorePath, "utf8")); + + // Memory-hogging function + const secretKeyBytes = await keystore.decrypt(definition.password); - decryptKeystores.queue( - definition, - (secretKeyBytes: Uint8Array) => { const signer: SignerLocal = { type: SignerType.Local, secretKey: bls.SecretKey.fromBytes(secretKeyBytes), @@ -76,19 +116,14 @@ export async function decryptKeystoreDefinitions( if (opts?.onDecrypt) { opts?.onDecrypt(index); } - }, - (error: Error) => { - // In-progress tasks can't be canceled, so there's a chance that multiple errors may be caught - // add to the list of errors - errors.push({keystoreFile: path.basename(definition.keystorePath), error}); - // cancel all pending tasks, no need to continue decrypting after we hit one error - decryptKeystores.cancel(); + } catch (e) { + errors.push({keystoreFile: path.basename(definition.keystorePath), error: e as Error}); + // stop processing, no need to continue decrypting after we hit one error + break; } - ); + } } - await decryptKeystores.completed(); - if (errors.length > 0) { // If an error occurs, the program isn't going to be running, // so we should unlock all lockfiles we created diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index 08548edb1072..fc7cb197c5aa 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -55,6 +55,7 @@ export type IValidatorCliArgs = AccountValidatorArgs & importKeystores?: string[]; importKeystoresPassword?: string; + disableKeystoresThreadPool?: boolean; "http.requestWireFormat"?: string; "http.responseWireFormat"?: string; @@ -301,6 +302,13 @@ export const validatorOptions: CliCommandOptions = { type: "string", }, + disableKeystoresThreadPool: { + hidden: true, + description: + "Disable thread pool and instead use main thread to decrypt keystores. This can speed up decryption in testing environments like Kurtosis", + type: "boolean", + }, + doppelgangerProtection: { alias: ["doppelgangerProtectionEnabled"], description: "Enables Doppelganger protection", diff --git a/packages/cli/src/cmds/validator/signers/index.ts b/packages/cli/src/cmds/validator/signers/index.ts index be028461c0ee..95daf7e69b0d 100644 --- a/packages/cli/src/cmds/validator/signers/index.ts +++ b/packages/cli/src/cmds/validator/signers/index.ts @@ -99,6 +99,7 @@ export async function getSignersFromArgs( ignoreLockFile: args.force, onDecrypt: needle, cacheFilePath: path.join(accountPaths.cacheDir, "imported_keystores.cache"), + disableThreadPool: args["disableKeystoresThreadPool"], logger, signal, }); @@ -133,6 +134,7 @@ export async function getSignersFromArgs( ignoreLockFile: args.force, onDecrypt: needle, cacheFilePath: path.join(accountPaths.cacheDir, "local_keystores.cache"), + disableThreadPool: args["disableKeystoresThreadPool"], logger, signal, }); diff --git a/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts b/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts index 8f1a82a55c3e..10f6b34bd152 100644 --- a/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts +++ b/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts @@ -5,7 +5,10 @@ import {rimraf} from "rimraf"; import {getKeystoresStr} from "@lodestar/test-utils"; import {cachedSeckeysHex} from "../../utils/cachedKeys.js"; import {testFilesDir} from "../../utils.js"; -import {decryptKeystoreDefinitions} from "../../../src/cmds/validator/keymanager/decryptKeystoreDefinitions.js"; +import { + decryptKeystoreDefinitions, + KeystoreDecryptOptions, +} from "../../../src/cmds/validator/keymanager/decryptKeystoreDefinitions.js"; import {LocalKeystoreDefinition} from "../../../src/cmds/validator/keymanager/interface.js"; import {LockfileError, unlockFilepath} from "../../../src/util/lockfile.js"; @@ -56,16 +59,20 @@ describe("decryptKeystoreDefinitions", () => { } }); - testDecryptKeystoreDefinitions(cacheFilePath); + testDecryptKeystoreDefinitions({cacheFilePath}); }); describe("without keystore cache", () => { testDecryptKeystoreDefinitions(); }); - function testDecryptKeystoreDefinitions(cacheFilePath?: string): void { + describe("disabled thread pool", () => { + testDecryptKeystoreDefinitions({disableThreadPool: true}); + }); + + function testDecryptKeystoreDefinitions(opts?: Partial): void { it("decrypt keystores", async () => { - const signers = await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath}); + const signers = await decryptKeystoreDefinitions(definitions, {logger: console, signal, ...opts}); expect(signers.length).toBe(secretKeys.length); for (const signer of signers) { const hexSecret = signer.secretKey.toHex(); @@ -75,11 +82,11 @@ describe("decryptKeystoreDefinitions", () => { }); it("fail to decrypt keystores if lockfiles already exist", async () => { - await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath}); + await decryptKeystoreDefinitions(definitions, {logger: console, signal, ...opts}); // lockfiles should exist after the first run try { - await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath}); + await decryptKeystoreDefinitions(definitions, {logger: console, signal, ...opts}); expect.fail("Second decrypt should fail due to failure to get lockfile"); } catch (e) { expect((e as LockfileError).code).toBe("ELOCKED"); @@ -87,10 +94,10 @@ describe("decryptKeystoreDefinitions", () => { }); it("decrypt keystores if lockfiles already exist if ignoreLockFile=true", async () => { - await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath}); + await decryptKeystoreDefinitions(definitions, {logger: console, signal, ...opts}); // lockfiles should exist after the first run - await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath, ignoreLockFile: true}); + await decryptKeystoreDefinitions(definitions, {logger: console, signal, ...opts, ignoreLockFile: true}); }); } }); From 81f9d976a92ebcbb6b361a748f31d62728620528 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 23 Jul 2024 09:56:56 +0100 Subject: [PATCH 09/11] fix: ignore noisy eventstream errors due to DNS resolution errors (#6972) --- packages/api/src/beacon/client/events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/beacon/client/events.ts b/packages/api/src/beacon/client/events.ts index 35383083ee44..2d63925a738a 100644 --- a/packages/api/src/beacon/client/events.ts +++ b/packages/api/src/beacon/client/events.ts @@ -44,7 +44,7 @@ export function getClient(config: ChainForkConfig, baseUrl: string): ApiClient { const errEs = err as unknown as EventSourceError; // Ignore noisy errors due to beacon node being offline - if (!errEs.message?.includes("ECONNREFUSED")) { + if (!/ECONNREFUSED|EAI_AGAIN/.test(errEs.message ?? "")) { // If there is no message it likely indicates that the server closed the connection onError?.(new Error(errEs.message ?? "Server closed connection")); } From 27012f96ec70725d1440069a1c6ca0f0424ea8ab Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 23 Jul 2024 10:49:03 +0100 Subject: [PATCH 10/11] fix: return finalized as false if finalized epoch is genesis epoch (#6965) * fix: return finalized as false if called with genesis slot or epoch * Update genesis epoch / slot checks --- packages/beacon-node/src/chain/chain.ts | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 61f044d4f895..081de6f85063 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -36,7 +36,7 @@ import { import {CheckpointWithHex, ExecutionStatus, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice"; import {ProcessShutdownCallback} from "@lodestar/validator"; import {Logger, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toHex} from "@lodestar/utils"; -import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {ForkSeq, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js"; import {IBeaconDb} from "../db/index.js"; @@ -430,7 +430,11 @@ export class BeaconChain implements IBeaconChain { {dontTransferCache: true}, RegenCaller.restApi ); - return {state, executionOptimistic: isOptimisticBlock(block), finalized: slot === finalizedBlock.slot}; + return { + state, + executionOptimistic: isOptimisticBlock(block), + finalized: slot === finalizedBlock.slot && finalizedBlock.slot !== GENESIS_SLOT, + }; } else { // Just check if state is already in the cache. If it's not dialed to the correct slot, // do not bother in advancing the state. restApiCanTriggerRegen == false means do no work @@ -440,7 +444,13 @@ export class BeaconChain implements IBeaconChain { } const state = this.regen.getStateSync(block.stateRoot); - return state && {state, executionOptimistic: isOptimisticBlock(block), finalized: slot === finalizedBlock.slot}; + return ( + state && { + state, + executionOptimistic: isOptimisticBlock(block), + finalized: slot === finalizedBlock.slot && finalizedBlock.slot !== GENESIS_SLOT, + } + ); } } else { // request for finalized state @@ -458,10 +468,11 @@ export class BeaconChain implements IBeaconChain { if (opts?.allowRegen) { const state = await this.regen.getState(stateRoot, RegenCaller.restApi); const block = this.forkChoice.getBlock(state.latestBlockHeader.hashTreeRoot()); + const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; return { state, executionOptimistic: block != null && isOptimisticBlock(block), - finalized: state.epochCtx.epoch <= this.forkChoice.getFinalizedCheckpoint().epoch, + finalized: state.epochCtx.epoch <= finalizedEpoch && finalizedEpoch !== GENESIS_EPOCH, }; } @@ -473,10 +484,11 @@ export class BeaconChain implements IBeaconChain { const cachedStateCtx = this.regen.getStateSync(stateRoot); if (cachedStateCtx) { const block = this.forkChoice.getBlock(cachedStateCtx.latestBlockHeader.hashTreeRoot()); + const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; return { state: cachedStateCtx, executionOptimistic: block != null && isOptimisticBlock(block), - finalized: cachedStateCtx.epochCtx.epoch <= this.forkChoice.getFinalizedCheckpoint().epoch, + finalized: cachedStateCtx.epochCtx.epoch <= finalizedEpoch && finalizedEpoch !== GENESIS_EPOCH, }; } @@ -491,10 +503,11 @@ export class BeaconChain implements IBeaconChain { const cachedStateCtx = this.regen.getCheckpointStateSync(checkpoint); if (cachedStateCtx) { const block = this.forkChoice.getBlock(cachedStateCtx.latestBlockHeader.hashTreeRoot()); + const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; return { state: cachedStateCtx, executionOptimistic: block != null && isOptimisticBlock(block), - finalized: cachedStateCtx.epochCtx.epoch <= this.forkChoice.getFinalizedCheckpoint().epoch, + finalized: cachedStateCtx.epochCtx.epoch <= finalizedEpoch && finalizedEpoch !== GENESIS_EPOCH, }; } From a187851b6a7b1b9be446f71812401d16312719da Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 24 Jul 2024 08:39:58 +0100 Subject: [PATCH 11/11] fix: start clock last when initializing validator client (#6973) --- packages/validator/src/validator.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 706cf7410b43..785d3de1db2a 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -128,23 +128,17 @@ export class Validator { if (opts.closed) { this.state = Status.closed; } else { - // "start" the validator - // Instantiates block and attestation services and runs them once the chain has been started. - this.state = Status.running; - this.clock.start(this.controller.signal); - this.chainHeaderTracker.start(this.controller.signal); - // Add notifier to warn user if primary node is unhealthy as there might // not be any errors in the logs due to fallback nodes handling the requests const {httpClient} = this.api; if (httpClient.urlsInits.length > 1) { const primaryNodeUrl = toSafePrintableUrl(httpClient.urlsInits[0].baseUrl); - this.clock?.runEveryEpoch(async () => { + this.clock.runEveryEpoch(async () => { // Only emit warning if URL score is 0 to prevent false positives // if just a single request fails which might happen due to other reasons if (httpClient.urlsScore[0] === 0) { - this.logger?.warn("Primary beacon node is unhealthy", {url: primaryNodeUrl}); + this.logger.warn("Primary beacon node is unhealthy", {url: primaryNodeUrl}); } }); } @@ -158,6 +152,11 @@ export class Validator { .catch((e) => this.logger.error("Error on fetchBeaconHealth", {}, e)) ); } + + // "start" the validator + this.state = Status.running; + this.clock.start(this.controller.signal); + this.chainHeaderTracker.start(this.controller.signal); } } @@ -178,8 +177,6 @@ export class Validator { let api: ApiClient; const {clientOrUrls, globalInit} = opts.api; if (typeof clientOrUrls === "string" || Array.isArray(clientOrUrls)) { - // This new api instance can make do with default timeout as a faster timeout is - // not necessary since this instance won't be used for validator duties api = getClient( { urls: typeof clientOrUrls === "string" ? [clientOrUrls] : clientOrUrls, @@ -329,6 +326,7 @@ export class Validator { strictFeeRecipientCheck, }); + // Instantiates block and attestation services and runs them once the chain has been started. return Validator.init(opts, genesis, metrics); }