From 9a8a6f4824fdb2eb36a06fc237ce8663bad412b8 Mon Sep 17 00:00:00 2001 From: g11tech Date: Mon, 19 Jun 2023 08:40:26 +0530 Subject: [PATCH 01/96] feat: publish blobsidecars instead of blobssidecar (#5662) * feat: publish blobsidecars instead of blobssidecar * add comment --- .../src/api/impl/beacon/blocks/index.ts | 20 +++++++-------- .../src/network/gossip/interface.ts | 4 +++ .../beacon-node/src/network/gossip/topic.ts | 12 +++++++++ packages/beacon-node/src/network/interface.ts | 4 +-- packages/beacon-node/src/network/network.ts | 25 +++++-------------- .../src/network/processor/gossipHandlers.ts | 4 +++ .../src/network/processor/gossipQueues.ts | 6 +++++ .../src/network/processor/index.ts | 1 + .../test/unit/network/gossip/topic.test.ts | 6 +++++ 9 files changed, 49 insertions(+), 33 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 9e7ee48579d9..aee1ddfc2ce2 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -208,20 +208,17 @@ export function getBeaconBlockApi({ let blockForImport: BlockInput, signedBlock: allForks.SignedBeaconBlock, signedBlobs: deneb.SignedBlobSidecars; if (isSignedBlockContents(signedBlockOrContents)) { - // Build a blockInput for post deneb, signedBlobs will be be used in followup PRs ({signedBlock, signedBlobSidecars: signedBlobs} = signedBlockOrContents); - const blobsSidecar = blobSidecarsToBlobsSidecar( - config, - signedBlock, - signedBlobs.map(({message}) => message) - ); - blockForImport = getBlockInput.postDeneb( config, signedBlock, BlockSource.api, // The blobsSidecar will be replaced in the followup PRs with just blobs - blobsSidecar + blobSidecarsToBlobsSidecar( + config, + signedBlock, + signedBlobs.map((sblob) => sblob.message) + ) ); } else { signedBlock = signedBlockOrContents; @@ -231,7 +228,8 @@ export function getBeaconBlockApi({ // Simple implementation of a pending block queue. Keeping the block here recycles the API logic, and keeps the // REST request promise without any extra infrastructure. - const msToBlockSlot = computeTimeAtSlot(config, signedBlock.message.slot, chain.genesisTime) * 1000 - Date.now(); + const msToBlockSlot = + computeTimeAtSlot(config, blockForImport.block.message.slot, chain.genesisTime) * 1000 - Date.now(); if (msToBlockSlot <= MAX_API_CLOCK_DISPARITY_MS && msToBlockSlot > 0) { // If block is a bit early, hold it in a promise. Equivalent to a pending queue. await sleep(msToBlockSlot); @@ -242,7 +240,7 @@ export function getBeaconBlockApi({ const publishPromises = [ // Send the block, regardless of whether or not it is valid. The API // specification is very clear that this is the desired behaviour. - () => network.publishBeaconBlockMaybeBlobs(blockForImport) as Promise, + () => network.publishBeaconBlock(signedBlock) as Promise, () => // there is no rush to persist block since we published it to gossip anyway chain.processBlock(blockForImport, {...opts, eagerPersistBlock: false}).catch((e) => { @@ -254,7 +252,7 @@ export function getBeaconBlockApi({ } throw e; }), - // TODO deneb: publish signed blobs as well + ...signedBlobs.map((signedBlob) => () => network.publishBlobSidecar(signedBlob)), ]; await promiseAllMaybeAsync(publishPromises); }, diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index e77205de01d9..dc178e8ee246 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -10,6 +10,7 @@ import {JobItemQueue} from "../../util/queue/index.js"; export enum GossipType { beacon_block = "beacon_block", + blob_sidecar = "blob_sidecar", beacon_block_and_blobs_sidecar = "beacon_block_and_blobs_sidecar", beacon_aggregate_and_proof = "beacon_aggregate_and_proof", beacon_attestation = "beacon_attestation", @@ -38,6 +39,7 @@ export interface IGossipTopic { export type GossipTopicTypeMap = { [GossipType.beacon_block]: {type: GossipType.beacon_block}; + [GossipType.blob_sidecar]: {type: GossipType.blob_sidecar; index: number}; [GossipType.beacon_block_and_blobs_sidecar]: {type: GossipType.beacon_block_and_blobs_sidecar}; [GossipType.beacon_aggregate_and_proof]: {type: GossipType.beacon_aggregate_and_proof}; [GossipType.beacon_attestation]: {type: GossipType.beacon_attestation; subnet: number}; @@ -68,6 +70,7 @@ export type SSZTypeOfGossipTopic = T extends {type: infer export type GossipTypeMap = { [GossipType.beacon_block]: allForks.SignedBeaconBlock; + [GossipType.blob_sidecar]: deneb.SignedBlobSidecar; [GossipType.beacon_block_and_blobs_sidecar]: deneb.SignedBeaconBlockAndBlobsSidecar; [GossipType.beacon_aggregate_and_proof]: phase0.SignedAggregateAndProof; [GossipType.beacon_attestation]: phase0.Attestation; @@ -83,6 +86,7 @@ export type GossipTypeMap = { export type GossipFnByType = { [GossipType.beacon_block]: (signedBlock: allForks.SignedBeaconBlock) => Promise | void; + [GossipType.blob_sidecar]: (signedBlobSidecar: deneb.SignedBlobSidecar) => Promise | void; [GossipType.beacon_block_and_blobs_sidecar]: ( signedBeaconBlockAndBlobsSidecar: deneb.SignedBeaconBlockAndBlobsSidecar ) => Promise | void; diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index 6cbf43bf66c9..8c5a35e74cfd 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -73,6 +73,8 @@ function stringifyGossipTopicType(topic: GossipTopic): string { case GossipType.beacon_attestation: case GossipType.sync_committee: return `${topic.type}_${topic.subnet}`; + case GossipType.blob_sidecar: + return `${topic.type}_${topic.index}`; } } @@ -82,6 +84,8 @@ export function getGossipSSZType(topic: GossipTopic) { case GossipType.beacon_block: // beacon_block is updated in altair to support the updated SignedBeaconBlock type return ssz[topic.fork].SignedBeaconBlock; + case GossipType.blob_sidecar: + return ssz.deneb.SignedBlobSidecar; case GossipType.beacon_block_and_blobs_sidecar: return ssz.deneb.SignedBeaconBlockAndBlobsSidecar; case GossipType.beacon_aggregate_and_proof: @@ -181,6 +185,13 @@ export function parseGossipTopic(forkDigestContext: ForkDigestContext, topicStr: } } + if (gossipTypeStr.startsWith(GossipType.blob_sidecar)) { + const indexStr = gossipTypeStr.slice(GossipType.blob_sidecar.length + 1); // +1 for '_' concatenating the topic name and the index + const index = parseInt(indexStr, 10); + if (Number.isNaN(index)) throw Error(`index ${indexStr} is not a number`); + return {type: GossipType.blob_sidecar, index, fork, encoding}; + } + throw Error(`Unknown gossip type ${gossipTypeStr}`); } catch (e) { (e as Error).message = `Invalid gossip topic ${topicStr}: ${(e as Error).message}`; @@ -253,6 +264,7 @@ function parseEncodingStr(encodingStr: string): GossipEncoding { // TODO: Review which yes, and which not export const gossipTopicIgnoreDuplicatePublishError: Record = { [GossipType.beacon_block]: true, + [GossipType.blob_sidecar]: true, [GossipType.beacon_block_and_blobs_sidecar]: true, [GossipType.beacon_aggregate_and_proof]: true, [GossipType.beacon_attestation]: true, diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index 705e71ab6453..12c327f1108b 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -3,7 +3,6 @@ import {Connection} from "@libp2p/interface-connection"; import {Registrar} from "@libp2p/interface-registrar"; import {ConnectionManager} from "@libp2p/interface-connection-manager"; import {Slot, SlotRootHex, allForks, altair, capella, deneb, phase0} from "@lodestar/types"; -import {BlockInput} from "../chain/blocks/types.js"; import {PeerIdStr} from "../util/peerId.js"; import {INetworkEventBus} from "./events.js"; import {INetworkCorePublic} from "./core/types.js"; @@ -44,9 +43,8 @@ export interface INetwork extends INetworkCorePublic { sendBlobSidecarsByRoot(peerId: PeerIdStr, request: deneb.BlobSidecarsByRootRequest): Promise; // Gossip - publishBeaconBlockMaybeBlobs(blockInput: BlockInput): Promise; publishBeaconBlock(signedBlock: allForks.SignedBeaconBlock): Promise; - publishSignedBeaconBlockAndBlobsSidecar(item: deneb.SignedBeaconBlockAndBlobsSidecar): Promise; + publishBlobSidecar(signedBlobSidecar: deneb.SignedBlobSidecar): Promise; publishBeaconAggregateAndProof(aggregateAndProof: phase0.SignedAggregateAndProof): Promise; publishBeaconAttestation(attestation: phase0.Attestation, subnet: number): Promise; publishVoluntaryExit(voluntaryExit: phase0.SignedVoluntaryExit): Promise; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index e4cfbee6f149..b07d01c3e083 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -14,7 +14,6 @@ import {IBeaconChain} from "../chain/index.js"; import {IBeaconDb} from "../db/interface.js"; import {PeerIdStr, peerIdToString} from "../util/peerId.js"; import {IClock} from "../util/clock.js"; -import {BlockInput, BlockInputType} from "../chain/blocks/types.js"; import {NetworkOptions} from "./options.js"; import {INetwork} from "./interface.js"; import {ReqRespMethod} from "./reqresp/index.js"; @@ -276,19 +275,6 @@ export class Network implements INetwork { // Gossip - async publishBeaconBlockMaybeBlobs(blockInput: BlockInput): Promise { - switch (blockInput.type) { - case BlockInputType.preDeneb: - return this.publishBeaconBlock(blockInput.block); - - case BlockInputType.postDeneb: - return this.publishSignedBeaconBlockAndBlobsSidecar({ - beaconBlock: blockInput.block as deneb.SignedBeaconBlock, - blobsSidecar: blockInput.blobs, - }); - } - } - async publishBeaconBlock(signedBlock: allForks.SignedBeaconBlock): Promise { const fork = this.config.getForkName(signedBlock.message.slot); return this.publishGossip({type: GossipType.beacon_block, fork}, signedBlock, { @@ -296,11 +282,12 @@ export class Network implements INetwork { }); } - async publishSignedBeaconBlockAndBlobsSidecar(item: deneb.SignedBeaconBlockAndBlobsSidecar): Promise { - const fork = this.config.getForkName(item.beaconBlock.message.slot); - return this.publishGossip( - {type: GossipType.beacon_block_and_blobs_sidecar, fork}, - item, + async publishBlobSidecar(signedBlobSidecar: deneb.SignedBlobSidecar): Promise { + const fork = this.config.getForkName(signedBlobSidecar.message.slot); + const index = signedBlobSidecar.message.index; + return this.publishGossip( + {type: GossipType.blob_sidecar, fork, index}, + signedBlobSidecar, {ignoreDuplicatePublishError: true} ); } diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 5804eceb6144..98fb43c5ff60 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -190,6 +190,10 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH handleValidBeaconBlock({...blockInput, serializedData}, peerIdStr, seenTimestampSec); }, + [GossipType.blob_sidecar]: async (_data, _topic, _peerIdStr, _seenTimestampSec) => { + // TODO DENEB: impl to be added on migration of blockinput + }, + [GossipType.beacon_block_and_blobs_sidecar]: async ({serializedData}, topic, peerIdStr, seenTimestampSec) => { const blockAndBlocks = sszDeserialize(topic, serializedData); const {beaconBlock, blobsSidecar} = blockAndBlocks; diff --git a/packages/beacon-node/src/network/processor/gossipQueues.ts b/packages/beacon-node/src/network/processor/gossipQueues.ts index ace41af43b47..63abd16ad3a8 100644 --- a/packages/beacon-node/src/network/processor/gossipQueues.ts +++ b/packages/beacon-node/src/network/processor/gossipQueues.ts @@ -35,6 +35,12 @@ const gossipQueueOpts: { } = { // validation gossip block asap [GossipType.beacon_block]: {maxLength: 1024, type: QueueType.FIFO, dropOpts: {type: DropType.count, count: 1}}, + // gossip length for blob is beacon block length * max blobs per block = 4096 + [GossipType.blob_sidecar]: { + maxLength: 4096, + type: QueueType.FIFO, + dropOpts: {type: DropType.count, count: 1}, + }, // TODO DENEB: What's a good queue max given that now blocks are much bigger? [GossipType.beacon_block_and_blobs_sidecar]: { maxLength: 32, diff --git a/packages/beacon-node/src/network/processor/index.ts b/packages/beacon-node/src/network/processor/index.ts index 04f2dc480224..9d5d05e7a672 100644 --- a/packages/beacon-node/src/network/processor/index.ts +++ b/packages/beacon-node/src/network/processor/index.ts @@ -56,6 +56,7 @@ type WorkOpts = { */ const executeGossipWorkOrderObj: Record = { [GossipType.beacon_block]: {bypassQueue: true}, + [GossipType.blob_sidecar]: {bypassQueue: true}, [GossipType.beacon_block_and_blobs_sidecar]: {bypassQueue: true}, [GossipType.beacon_aggregate_and_proof]: {}, [GossipType.voluntary_exit]: {}, diff --git a/packages/beacon-node/test/unit/network/gossip/topic.test.ts b/packages/beacon-node/test/unit/network/gossip/topic.test.ts index 7eb037a8fe43..c2e6bb5c8015 100644 --- a/packages/beacon-node/test/unit/network/gossip/topic.test.ts +++ b/packages/beacon-node/test/unit/network/gossip/topic.test.ts @@ -15,6 +15,12 @@ describe("network / gossip / topic", function () { topicStr: "/eth2/18ae4ccb/beacon_block/ssz_snappy", }, ], + [GossipType.blob_sidecar]: [ + { + topic: {type: GossipType.blob_sidecar, index: 1, fork: ForkName.deneb, encoding}, + topicStr: "/eth2/46acb19a/blob_sidecar_1/ssz_snappy", + }, + ], [GossipType.beacon_block_and_blobs_sidecar]: [ { topic: {type: GossipType.beacon_block_and_blobs_sidecar, fork: ForkName.deneb, encoding}, From 0122664d8ec3dc79e44947be962efc83f21fb91b Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 19 Jun 2023 10:00:50 +0200 Subject: [PATCH 02/96] fix(validator): abortable doppelganger detection (#5665) --- packages/validator/src/services/doppelgangerService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validator/src/services/doppelgangerService.ts b/packages/validator/src/services/doppelgangerService.ts index ddfe2172df2c..7f656bf9aa2c 100644 --- a/packages/validator/src/services/doppelgangerService.ts +++ b/packages/validator/src/services/doppelgangerService.ts @@ -83,7 +83,7 @@ export class DoppelgangerService { return getStatus(this.doppelgangerStateByPubkey.get(pubKeyHex)) === DoppelgangerStatus.VerifiedSafe; } - private pollLiveness = async (currentEpoch: Epoch): Promise => { + private pollLiveness = async (currentEpoch: Epoch, signal: AbortSignal): Promise => { if (currentEpoch < 0) { return; } @@ -91,7 +91,7 @@ export class DoppelgangerService { const endSlotOfCurrentEpoch = computeStartSlotAtEpoch(currentEpoch + 1) - 1; // Run the doppelganger protection check 75% through the last slot of this epoch. This // *should* mean that the BN has seen the blocks and attestations for the epoch - await sleep(this.clock.msToSlot(endSlotOfCurrentEpoch + 3 / 4)); + await sleep(this.clock.msToSlot(endSlotOfCurrentEpoch + 3 / 4), signal); // Collect indices that still need doppelganger checks const pubkeysToCheckWithoutIndex: PubkeyHex[] = []; From fd0f476523772c307ee2d8b73399510e72aa72f5 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 19 Jun 2023 10:04:10 +0200 Subject: [PATCH 03/96] fix(validator): abortable sleep in prepareForNextEpoch (#5666) --- packages/validator/src/services/attestationDuties.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/validator/src/services/attestationDuties.ts b/packages/validator/src/services/attestationDuties.ts index 916c02f91994..8762c4258ce0 100644 --- a/packages/validator/src/services/attestationDuties.ts +++ b/packages/validator/src/services/attestationDuties.ts @@ -108,14 +108,14 @@ export class AttestationDutiesService { * If a reorg dependent root comes at a slot other than last slot of epoch * just update this.pendingDependentRootByEpoch() and process here */ - private prepareForNextEpoch = async (slot: Slot): Promise => { + private prepareForNextEpoch = async (slot: Slot, signal: AbortSignal): Promise => { // only interested in last slot of epoch if ((slot + 1) % SLOTS_PER_EPOCH !== 0) { return; } // during the 1 / 3 of epoch, last block of epoch may come - await sleep(this.clock.msToSlot(slot + 1 / 3)); + await sleep(this.clock.msToSlot(slot + 1 / 3), signal); const nextEpoch = computeEpochAtSlot(slot) + 1; const dependentRoot = this.dutiesByIndexByEpoch.get(nextEpoch)?.dependentRoot; From 569933316eca77c26de354c590c5ce63ae3dbbaf Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Mon, 19 Jun 2023 15:24:34 +0700 Subject: [PATCH 04/96] fix: gossipsub to yield more to the macro queue (#5664) * fix: add setTimeout to onGossipsubMessage and onValidationResult * chore: more comments --- .../src/network/gossip/gossipsub.ts | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/beacon-node/src/network/gossip/gossipsub.ts b/packages/beacon-node/src/network/gossip/gossipsub.ts index fbafabcf1a44..ef159e019077 100644 --- a/packages/beacon-node/src/network/gossip/gossipsub.ts +++ b/packages/beacon-node/src/network/gossip/gossipsub.ts @@ -279,21 +279,30 @@ export class Eth2Gossipsub extends GossipSub { // Get seenTimestamp before adding the message to the queue or add async delays const seenTimestampSec = Date.now() / 1000; - // Emit message to network processor - this.events.emit(NetworkEvent.pendingGossipsubMessage, { - topic, - msg, - msgId, - // Hot path, use cached .toString() version - propagationSource: propagationSource.toString(), - seenTimestampSec, - startProcessUnixSec: null, - }); + // Use setTimeout to yield to the macro queue + // Without this we'll have huge event loop lag + // See https://github.com/ChainSafe/lodestar/issues/5604 + setTimeout(() => { + this.events.emit(NetworkEvent.pendingGossipsubMessage, { + topic, + msg, + msgId, + // Hot path, use cached .toString() version + propagationSource: propagationSource.toString(), + seenTimestampSec, + startProcessUnixSec: null, + }); + }, 0); } private onValidationResult(data: NetworkEventData[NetworkEvent.gossipMessageValidationResult]): void { - // TODO: reportMessageValidationResult should take PeerIdStr since it only uses string version - this.reportMessageValidationResult(data.msgId, peerIdFromString(data.propagationSource), data.acceptance); + // Use setTimeout to yield to the macro queue + // Without this we'll have huge event loop lag + // See https://github.com/ChainSafe/lodestar/issues/5604 + setTimeout(() => { + // TODO: reportMessageValidationResult should take PeerIdStr since it only uses string version + this.reportMessageValidationResult(data.msgId, peerIdFromString(data.propagationSource), data.acceptance); + }, 0); } } From c8f94b7f35a5b416f0ec508faf80461b54fd6804 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 19 Jun 2023 11:31:21 +0300 Subject: [PATCH 05/96] feat: skip serializing block after fetching from network (#5573) * Skip serializing block after fetching from network * fix: beaconBlocksMaybeBlobsByRange.test.ts --------- Co-authored-by: Tuyen Nguyen --- .../src/api/impl/beacon/blocks/index.ts | 6 ++-- .../beacon-node/src/chain/blocks/index.ts | 10 +++--- .../beacon-node/src/chain/blocks/types.ts | 23 +++++++++----- .../src/chain/blocks/verifyBlock.ts | 4 +-- .../chain/blocks/verifyBlocksSanityChecks.ts | 8 ++--- .../src/chain/blocks/writeBlockInputToDb.ts | 18 ++++------- packages/beacon-node/src/chain/chain.ts | 5 ++- packages/beacon-node/src/chain/interface.ts | 18 ++--------- packages/beacon-node/src/network/interface.ts | 6 ++-- packages/beacon-node/src/network/network.ts | 14 ++++++--- .../src/network/processor/gossipHandlers.ts | 17 +++++----- .../reqresp/beaconBlocksMaybeBlobsByRange.ts | 23 +++++++------- .../reqresp/beaconBlocksMaybeBlobsByRoot.ts | 4 +-- .../src/network/reqresp/utils/collect.ts | 27 ++++++++++++++++ .../utils/collectSequentialBlocksInRange.ts | 9 +++--- .../beacon-node/src/sync/backfill/backfill.ts | 31 ++++++++++++------- .../beacon-node/src/sync/backfill/verify.ts | 19 ++++++------ .../onWorker/dataSerialization.test.ts | 1 + .../test/e2e/network/reqresp.test.ts | 5 ++- .../test/e2e/sync/unknownBlockSync.test.ts | 2 +- .../perf/chain/verifyImportBlocks.test.ts | 2 +- .../test/spec/presets/fork_choice.ts | 5 +-- .../blocks/verifyBlocksSanityChecks.test.ts | 2 +- .../beaconBlocksMaybeBlobsByRange.test.ts | 12 +++++-- .../test/unit/sync/backfill/verify.test.ts | 12 ++++--- .../test/unit/sync/range/batch.test.ts | 2 +- .../test/unit/sync/range/chain.test.ts | 6 ++-- .../test/unit/sync/unknownBlock.test.ts | 10 ++++-- packages/types/src/types.ts | 2 -- 29 files changed, 177 insertions(+), 126 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index aee1ddfc2ce2..7833775b4c9a 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -218,12 +218,14 @@ export function getBeaconBlockApi({ config, signedBlock, signedBlobs.map((sblob) => sblob.message) - ) + ), + null ); } else { signedBlock = signedBlockOrContents; signedBlobs = []; - blockForImport = getBlockInput.preDeneb(config, signedBlock, BlockSource.api); + // TODO: Once API supports submitting data as SSZ, replace null with blockBytes + blockForImport = getBlockInput.preDeneb(config, signedBlock, BlockSource.api, null); } // Simple implementation of a pending block queue. Keeping the block here recycles the API logic, and keeps the diff --git a/packages/beacon-node/src/chain/blocks/index.ts b/packages/beacon-node/src/chain/blocks/index.ts index a9eba7f518bf..46e16e2317e7 100644 --- a/packages/beacon-node/src/chain/blocks/index.ts +++ b/packages/beacon-node/src/chain/blocks/index.ts @@ -1,4 +1,4 @@ -import {WithOptionalBytes, allForks} from "@lodestar/types"; +import {allForks} from "@lodestar/types"; import {toHex, isErrorAborted} from "@lodestar/utils"; import {JobItemQueue, isQueueErrorAborted} from "../../util/queue/index.js"; import {Metrics} from "../../metrics/metrics.js"; @@ -19,10 +19,10 @@ const QUEUE_MAX_LENGTH = 256; * BlockProcessor processes block jobs in a queued fashion, one after the other. */ export class BlockProcessor { - readonly jobQueue: JobItemQueue<[WithOptionalBytes[], ImportBlockOpts], void>; + readonly jobQueue: JobItemQueue<[BlockInput[], ImportBlockOpts], void>; constructor(chain: BeaconChain, metrics: Metrics | null, opts: BlockProcessOpts, signal: AbortSignal) { - this.jobQueue = new JobItemQueue<[WithOptionalBytes[], ImportBlockOpts], void>( + this.jobQueue = new JobItemQueue<[BlockInput[], ImportBlockOpts], void>( (job, importOpts) => { return processBlocks.call(chain, job, {...opts, ...importOpts}); }, @@ -31,7 +31,7 @@ export class BlockProcessor { ); } - async processBlocksJob(job: WithOptionalBytes[], opts: ImportBlockOpts = {}): Promise { + async processBlocksJob(job: BlockInput[], opts: ImportBlockOpts = {}): Promise { await this.jobQueue.push(job, opts); } } @@ -48,7 +48,7 @@ export class BlockProcessor { */ export async function processBlocks( this: BeaconChain, - blocks: WithOptionalBytes[], + blocks: BlockInput[], opts: BlockProcessOpts & ImportBlockOpts ): Promise { if (blocks.length === 0) { diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index 2b1c521000ef..affbbbe5fd9e 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -1,6 +1,6 @@ import {CachedBeaconStateAllForks, computeEpochAtSlot, DataAvailableStatus} from "@lodestar/state-transition"; import {MaybeValidExecutionStatus} from "@lodestar/fork-choice"; -import {allForks, deneb, Slot, WithOptionalBytes} from "@lodestar/types"; +import {allForks, deneb, Slot} from "@lodestar/types"; import {ForkSeq, MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; @@ -19,9 +19,10 @@ export enum BlockSource { byRoot = "req_resp_by_root", } -export type BlockInput = - | {type: BlockInputType.preDeneb; block: allForks.SignedBeaconBlock; source: BlockSource} - | {type: BlockInputType.postDeneb; block: allForks.SignedBeaconBlock; source: BlockSource; blobs: deneb.BlobsSidecar}; +export type BlockInput = {block: allForks.SignedBeaconBlock; source: BlockSource; blockBytes: Uint8Array | null} & ( + | {type: BlockInputType.preDeneb} + | {type: BlockInputType.postDeneb; blobs: deneb.BlobsSidecar} +); export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clockSlot: Slot): boolean { return ( @@ -51,7 +52,12 @@ export function blobSidecarsToBlobsSidecar( } export const getBlockInput = { - preDeneb(config: ChainForkConfig, block: allForks.SignedBeaconBlock, source: BlockSource): BlockInput { + preDeneb( + config: ChainForkConfig, + block: allForks.SignedBeaconBlock, + source: BlockSource, + blockBytes: Uint8Array | null + ): BlockInput { if (config.getForkSeq(block.message.slot) >= ForkSeq.deneb) { throw Error(`Post Deneb block slot ${block.message.slot}`); } @@ -59,6 +65,7 @@ export const getBlockInput = { type: BlockInputType.preDeneb, block, source, + blockBytes, }; }, @@ -66,7 +73,8 @@ export const getBlockInput = { config: ChainForkConfig, block: allForks.SignedBeaconBlock, source: BlockSource, - blobs: deneb.BlobsSidecar + blobs: deneb.BlobsSidecar, + blockBytes: Uint8Array | null ): BlockInput { if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) { throw Error(`Pre Deneb block slot ${block.message.slot}`); @@ -76,6 +84,7 @@ export const getBlockInput = { block, source, blobs, + blockBytes, }; }, }; @@ -130,7 +139,7 @@ export type ImportBlockOpts = { * A wrapper around a `SignedBeaconBlock` that indicates that this block is fully verified and ready to import */ export type FullyVerifiedBlock = { - blockInput: WithOptionalBytes; + blockInput: BlockInput; postState: CachedBeaconStateAllForks; parentBlockSlot: Slot; proposerBalanceDelta: number; diff --git a/packages/beacon-node/src/chain/blocks/verifyBlock.ts b/packages/beacon-node/src/chain/blocks/verifyBlock.ts index 636424eac255..2435c0cc47e4 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlock.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlock.ts @@ -4,7 +4,7 @@ import { isStateValidatorsNodesPopulated, DataAvailableStatus, } from "@lodestar/state-transition"; -import {WithOptionalBytes, bellatrix} from "@lodestar/types"; +import {bellatrix} from "@lodestar/types"; import {ForkName} from "@lodestar/params"; import {toHexString} from "@chainsafe/ssz"; import {ProtoBlock} from "@lodestar/fork-choice"; @@ -37,7 +37,7 @@ import {writeBlockInputToDb} from "./writeBlockInputToDb.js"; export async function verifyBlocksInEpoch( this: BeaconChain, parentBlock: ProtoBlock, - blocksInput: WithOptionalBytes[], + blocksInput: BlockInput[], dataAvailabilityStatuses: DataAvailableStatus[], opts: BlockProcessOpts & ImportBlockOpts ): Promise<{ diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts index 7b0383bf4d13..7fa5bb6fa405 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts @@ -1,7 +1,7 @@ import {computeStartSlotAtEpoch, DataAvailableStatus} from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import {Slot, deneb, WithOptionalBytes} from "@lodestar/types"; +import {Slot, deneb} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {BlockError, BlockErrorCode} from "../errors/index.js"; @@ -22,10 +22,10 @@ import {BlockInput, BlockInputType, ImportBlockOpts} from "./types.js"; */ export function verifyBlocksSanityChecks( chain: {forkChoice: IForkChoice; clock: IClock; config: ChainForkConfig}, - blocks: WithOptionalBytes[], + blocks: BlockInput[], opts: ImportBlockOpts ): { - relevantBlocks: WithOptionalBytes[]; + relevantBlocks: BlockInput[]; dataAvailabilityStatuses: DataAvailableStatus[]; parentSlots: Slot[]; parentBlock: ProtoBlock | null; @@ -34,7 +34,7 @@ export function verifyBlocksSanityChecks( throw Error("Empty partiallyVerifiedBlocks"); } - const relevantBlocks: WithOptionalBytes[] = []; + const relevantBlocks: BlockInput[] = []; const dataAvailabilityStatuses: DataAvailableStatus[] = []; const parentSlots: Slot[] = []; let parentBlock: ProtoBlock | null = null; diff --git a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts index d5b928d131bf..ba6b0956b754 100644 --- a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts +++ b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts @@ -1,4 +1,4 @@ -import {WithOptionalBytes, allForks, deneb} from "@lodestar/types"; +import {allForks, deneb} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; import {BeaconChain} from "../chain.js"; import {BlockInput, BlockInputType} from "./types.js"; @@ -10,20 +10,17 @@ import {BlockInput, BlockInputType} from "./types.js"; * This operation may be performed before, during or after importing to the fork-choice. As long as errors * are handled properly for eventual consistency. */ -export async function writeBlockInputToDb( - this: BeaconChain, - blocksInput: WithOptionalBytes[] -): Promise { +export async function writeBlockInputToDb(this: BeaconChain, blocksInput: BlockInput[]): Promise { const fnPromises: Promise[] = []; for (const blockInput of blocksInput) { - const {block, serializedData, type} = blockInput; + const {block, blockBytes, type} = blockInput; const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); const blockRootHex = toHex(blockRoot); - if (serializedData) { + if (blockBytes) { // skip serializing data if we already have it this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc(); - fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), serializedData)); + fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes)); } else { this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc(); fnPromises.push(this.db.block.add(block)); @@ -51,10 +48,7 @@ export async function writeBlockInputToDb( /** * Prunes eagerly persisted block inputs only if not known to the fork-choice */ -export async function removeEagerlyPersistedBlockInputs( - this: BeaconChain, - blockInputs: WithOptionalBytes[] -): Promise { +export async function removeEagerlyPersistedBlockInputs(this: BeaconChain, blockInputs: BlockInput[]): Promise { const blockToRemove: allForks.SignedBeaconBlock[] = []; const blobsToRemove: deneb.BlobsSidecar[] = []; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index cdf19a3ea382..641bd7ac82c6 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -23,7 +23,6 @@ import { ValidatorIndex, deneb, Wei, - WithOptionalBytes, bellatrix, } from "@lodestar/types"; import {CheckpointWithHex, ExecutionStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; @@ -558,11 +557,11 @@ export class BeaconChain implements IBeaconChain { return blobSidecars; } - async processBlock(block: WithOptionalBytes, opts?: ImportBlockOpts): Promise { + async processBlock(block: BlockInput, opts?: ImportBlockOpts): Promise { return this.blockProcessor.processBlocksJob([block], opts); } - async processChainSegment(blocks: WithOptionalBytes[], opts?: ImportBlockOpts): Promise { + async processChainSegment(blocks: BlockInput[], opts?: ImportBlockOpts): Promise { return this.blockProcessor.processBlocksJob(blocks, opts); } diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 977aed236f57..5f80c1411220 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -1,16 +1,4 @@ -import { - allForks, - UintNum64, - Root, - phase0, - Slot, - RootHex, - Epoch, - ValidatorIndex, - deneb, - Wei, - WithOptionalBytes, -} from "@lodestar/types"; +import {allForks, UintNum64, Root, phase0, Slot, RootHex, Epoch, ValidatorIndex, deneb, Wei} from "@lodestar/types"; import { BeaconStateAllForks, CachedBeaconStateAllForks, @@ -152,9 +140,9 @@ export interface IBeaconChain { produceBlindedBlock(blockAttributes: BlockAttributes): Promise<{block: allForks.BlindedBeaconBlock; blockValue: Wei}>; /** Process a block until complete */ - processBlock(block: WithOptionalBytes, opts?: ImportBlockOpts): Promise; + processBlock(block: BlockInput, opts?: ImportBlockOpts): Promise; /** Process a chain of blocks until complete */ - processChainSegment(blocks: WithOptionalBytes[], opts?: ImportBlockOpts): Promise; + processChainSegment(blocks: BlockInput[], opts?: ImportBlockOpts): Promise; getStatus(): phase0.Status; diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index 12c327f1108b..5f7ec83c76d6 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -10,6 +10,8 @@ import {GossipType} from "./gossip/interface.js"; import {PendingGossipsubMessage} from "./processor/types.js"; import {PeerAction} from "./peers/index.js"; +export type WithBytes = {data: T; bytes: Uint8Array}; + /** * The architecture of the network looks like so: * - core: @@ -34,11 +36,11 @@ export interface INetwork extends INetworkCorePublic { sendBeaconBlocksByRange( peerId: PeerIdStr, request: phase0.BeaconBlocksByRangeRequest - ): Promise; + ): Promise[]>; sendBeaconBlocksByRoot( peerId: PeerIdStr, request: phase0.BeaconBlocksByRootRequest - ): Promise; + ): Promise[]>; sendBlobSidecarsByRange(peerId: PeerIdStr, request: deneb.BlobSidecarsByRangeRequest): Promise; sendBlobSidecarsByRoot(peerId: PeerIdStr, request: deneb.BlobSidecarsByRootRequest): Promise; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index b07d01c3e083..e0a2b9103148 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -15,7 +15,7 @@ import {IBeaconDb} from "../db/interface.js"; import {PeerIdStr, peerIdToString} from "../util/peerId.js"; import {IClock} from "../util/clock.js"; import {NetworkOptions} from "./options.js"; -import {INetwork} from "./interface.js"; +import {WithBytes, INetwork} from "./interface.js"; import {ReqRespMethod} from "./reqresp/index.js"; import {GossipHandlers, GossipTopicMap, GossipType, GossipTypeMap} from "./gossip/index.js"; import {PeerAction, PeerScoreStats} from "./peers/index.js"; @@ -24,7 +24,11 @@ import {CommitteeSubscription} from "./subnets/index.js"; import {isPublishToZeroPeersError} from "./util.js"; import {NetworkProcessor, PendingGossipsubMessage} from "./processor/index.js"; import {INetworkCore, NetworkCore, WorkerNetworkCore} from "./core/index.js"; -import {collectExactOneTyped, collectMaxResponseTyped} from "./reqresp/utils/collect.js"; +import { + collectExactOneTyped, + collectMaxResponseTyped, + collectMaxResponseTypedWithBytes, +} from "./reqresp/utils/collect.js"; import {GetReqRespHandlerFn, Version, requestSszTypeByMethod, responseSszTypeByMethod} from "./reqresp/types.js"; import {collectSequentialBlocksInRange} from "./reqresp/utils/collectSequentialBlocksInRange.js"; import {getGossipSSZType, gossipTopicIgnoreDuplicatePublishError, stringifyGossipTopic} from "./gossip/topic.js"; @@ -396,7 +400,7 @@ export class Network implements INetwork { async sendBeaconBlocksByRange( peerId: PeerIdStr, request: phase0.BeaconBlocksByRangeRequest - ): Promise { + ): Promise[]> { return collectSequentialBlocksInRange( this.sendReqRespRequest( peerId, @@ -412,8 +416,8 @@ export class Network implements INetwork { async sendBeaconBlocksByRoot( peerId: PeerIdStr, request: phase0.BeaconBlocksByRootRequest - ): Promise { - return collectMaxResponseTyped( + ): Promise[]> { + return collectMaxResponseTypedWithBytes( this.sendReqRespRequest( peerId, ReqRespMethod.BeaconBlocksByRoot, diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 98fb43c5ff60..72ae32cc8887 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -1,7 +1,7 @@ import {toHexString} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; import {Logger, prettyBytes} from "@lodestar/utils"; -import {Root, Slot, ssz, WithBytes} from "@lodestar/types"; +import {Root, Slot, ssz} from "@lodestar/types"; import {ForkName, ForkSeq} from "@lodestar/params"; import {Metrics} from "../../metrics/index.js"; import {OpSource} from "../../metrics/validatorMonitor.js"; @@ -125,11 +125,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } } - function handleValidBeaconBlock( - blockInput: WithBytes, - peerIdStr: string, - seenTimestampSec: number - ): void { + function handleValidBeaconBlock(blockInput: BlockInput, peerIdStr: string, seenTimestampSec: number): void { const signedBlock = blockInput.block; // Handler - MUST NOT `await`, to allow validation result to be propagated @@ -185,9 +181,9 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH throw new GossipActionError(GossipAction.REJECT, {code: "POST_DENEB_BLOCK"}); } - const blockInput = getBlockInput.preDeneb(config, signedBlock, BlockSource.gossip); + const blockInput = getBlockInput.preDeneb(config, signedBlock, BlockSource.gossip, serializedData); await validateBeaconBlock(blockInput, topic.fork, peerIdStr, seenTimestampSec); - handleValidBeaconBlock({...blockInput, serializedData}, peerIdStr, seenTimestampSec); + handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); }, [GossipType.blob_sidecar]: async (_data, _topic, _peerIdStr, _seenTimestampSec) => { @@ -203,10 +199,11 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } // Validate block + blob. Then forward, then handle both - const blockInput = getBlockInput.postDeneb(config, beaconBlock, BlockSource.gossip, blobsSidecar); + // TODO DENEB: replace null with proper binary data for block and blobs separately + const blockInput = getBlockInput.postDeneb(config, beaconBlock, BlockSource.gossip, blobsSidecar, null); await validateBeaconBlock(blockInput, topic.fork, peerIdStr, seenTimestampSec); validateGossipBlobsSidecar(beaconBlock, blobsSidecar); - handleValidBeaconBlock({...blockInput, serializedData}, peerIdStr, seenTimestampSec); + handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); }, [GossipType.beacon_aggregate_and_proof]: async ({serializedData}, topic, _peer, seenTimestampSec) => { diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts index c254127f5a98..7e8d4c972681 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts @@ -5,7 +5,7 @@ import {computeEpochAtSlot} from "@lodestar/state-transition"; import {BlockInput, BlockSource, getBlockInput, blobSidecarsToBlobsSidecar} from "../../chain/blocks/types.js"; import {PeerIdStr} from "../../util/peerId.js"; -import {INetwork} from "../interface.js"; +import {INetwork, WithBytes} from "../interface.js"; export async function beaconBlocksMaybeBlobsByRange( config: ChainForkConfig, @@ -33,7 +33,7 @@ export async function beaconBlocksMaybeBlobsByRange( // Note: Assumes all blocks in the same epoch if (config.getForkSeq(startSlot) < ForkSeq.deneb) { const blocks = await network.sendBeaconBlocksByRange(peerId, request); - return blocks.map((block) => getBlockInput.preDeneb(config, block, BlockSource.byRange)); + return blocks.map((block) => getBlockInput.preDeneb(config, block.data, BlockSource.byRange, block.bytes)); } // Only request blobs if they are recent enough @@ -55,7 +55,7 @@ export async function beaconBlocksMaybeBlobsByRange( // Assumes that the blobs are in the same sequence as blocks, doesn't require block to be sorted export function matchBlockWithBlobs( config: ChainForkConfig, - allBlocks: allForks.SignedBeaconBlock[], + allBlocks: WithBytes[], allBlobSidecars: deneb.BlobSidecar[], endSlot: Slot, blockSource: BlockSource @@ -72,29 +72,30 @@ export function matchBlockWithBlobs( // Assuming that the blocks and blobs will come in same sorted order for (let i = 0; i < allBlocks.length; i++) { const block = allBlocks[i]; - if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) { - blockInputs.push(getBlockInput.preDeneb(config, block, blockSource)); + if (config.getForkSeq(block.data.message.slot) < ForkSeq.deneb) { + blockInputs.push(getBlockInput.preDeneb(config, block.data, blockSource, block.bytes)); } else { const blobSidecars: deneb.BlobSidecar[] = []; let blobSidecar: deneb.BlobSidecar; - while ((blobSidecar = allBlobSidecars[blobSideCarIndex])?.slot === block.message.slot) { + while ((blobSidecar = allBlobSidecars[blobSideCarIndex])?.slot === block.data.message.slot) { blobSidecars.push(blobSidecar); - lastMatchedSlot = block.message.slot; + lastMatchedSlot = block.data.message.slot; blobSideCarIndex++; } // Quick inspect how many blobSidecars was expected - const blobKzgCommitmentsLen = (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; + const blobKzgCommitmentsLen = (block.data.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; if (blobKzgCommitmentsLen !== blobSidecars.length) { throw Error( - `Missing blobSidecars for blockSlot=${block.message.slot} with blobKzgCommitmentsLen=${blobKzgCommitmentsLen} blobSidecars=${blobSidecars.length}` + `Missing blobSidecars for blockSlot=${block.data.message.slot} with blobKzgCommitmentsLen=${blobKzgCommitmentsLen} blobSidecars=${blobSidecars.length}` ); } // TODO DENEB: cleanup blobSidecars to blobsSidecar conversion on migration of blockInput - const blobsSidecar = blobSidecarsToBlobsSidecar(config, block, blobSidecars); - blockInputs.push(getBlockInput.postDeneb(config, block, blockSource, blobsSidecar)); + const blobsSidecar = blobSidecarsToBlobsSidecar(config, block.data, blobSidecars); + // TODO DENEB: instead of null, pass payload in bytes + blockInputs.push(getBlockInput.postDeneb(config, block.data, blockSource, blobsSidecar, null)); } } diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts index 962ca7bb48dc..4811d8bae9e7 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts @@ -18,8 +18,8 @@ export async function beaconBlocksMaybeBlobsByRoot( const blobIdentifiers: deneb.BlobIdentifier[] = []; for (const block of allBlocks) { - const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); - const blobKzgCommitmentsLen = (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments?.length ?? 0; + const blockRoot = config.getForkTypes(block.data.message.slot).BeaconBlock.hashTreeRoot(block.data.message); + const blobKzgCommitmentsLen = (block.data.message.body as deneb.BeaconBlockBody).blobKzgCommitments?.length ?? 0; for (let index = 0; index < blobKzgCommitmentsLen; index++) { blobIdentifiers.push({blockRoot, index}); } diff --git a/packages/beacon-node/src/network/reqresp/utils/collect.ts b/packages/beacon-node/src/network/reqresp/utils/collect.ts index 241dc8e3d3ad..8786dfb6883f 100644 --- a/packages/beacon-node/src/network/reqresp/utils/collect.ts +++ b/packages/beacon-node/src/network/reqresp/utils/collect.ts @@ -1,6 +1,7 @@ import {ResponseIncoming, RequestErrorCode, RequestError} from "@lodestar/reqresp"; import {Type} from "@chainsafe/ssz"; import {ResponseTypeGetter} from "../types.js"; +import {WithBytes} from "../../interface.js"; /** * Sink for `*`, from @@ -47,6 +48,32 @@ export async function collectMaxResponseTyped( return responses; } +/** + * Sink for `*`, from + * ```bnf + * response ::= * + * ``` + * Collects a bounded list of responses up to `maxResponses` + */ +export async function collectMaxResponseTypedWithBytes( + source: AsyncIterable, + maxResponses: number, + typeFn: ResponseTypeGetter +): Promise[]> { + // else: zero or more responses + const responses: WithBytes[] = []; + for await (const chunk of source) { + const type = typeFn(chunk.fork, chunk.protocolVersion); + const data = sszDeserializeResponse(type, chunk.data); + responses.push({data, bytes: chunk.data}); + + if (maxResponses !== undefined && responses.length >= maxResponses) { + break; + } + } + return responses; +} + /** Light wrapper on type to wrap deserialize errors */ export function sszDeserializeResponse(type: Type, bytes: Uint8Array): T { try { diff --git a/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts b/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts index 153c60a5bcfc..e1a637c7df89 100644 --- a/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts +++ b/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts @@ -1,6 +1,7 @@ import {ResponseIncoming} from "@lodestar/reqresp"; import {allForks, phase0} from "@lodestar/types"; import {LodestarError} from "@lodestar/utils"; +import {WithBytes} from "../../interface.js"; import {ReqRespMethod, responseSszTypeByMethod} from "../types.js"; import {sszDeserializeResponse} from "./collect.js"; @@ -11,8 +12,8 @@ import {sszDeserializeResponse} from "./collect.js"; export async function collectSequentialBlocksInRange( blockStream: AsyncIterable, {count, startSlot}: Pick -): Promise { - const blocks: allForks.SignedBeaconBlock[] = []; +): Promise[]> { + const blocks: WithBytes[] = []; for await (const chunk of blockStream) { const blockType = responseSszTypeByMethod[ReqRespMethod.BeaconBlocksByRange](chunk.fork, chunk.protocolVersion); @@ -31,12 +32,12 @@ export async function collectSequentialBlocksInRange( const prevBlock = blocks.length === 0 ? null : blocks[blocks.length - 1]; if (prevBlock) { - if (prevBlock.message.slot >= blockSlot) { + if (prevBlock.data.message.slot >= blockSlot) { throw new BlocksByRangeError({code: BlocksByRangeErrorCode.BAD_SEQUENCE}); } } - blocks.push(block); + blocks.push({data: block, bytes: chunk.data}); if (blocks.length >= count) { break; // Done, collected all blocks } diff --git a/packages/beacon-node/src/sync/backfill/backfill.ts b/packages/beacon-node/src/sync/backfill/backfill.ts index 472bd6313cbd..a93edce53d0b 100644 --- a/packages/beacon-node/src/sync/backfill/backfill.ts +++ b/packages/beacon-node/src/sync/backfill/backfill.ts @@ -746,31 +746,32 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} } private async syncBlockByRoot(peer: PeerIdStr, anchorBlockRoot: Root): Promise { - const [anchorBlock] = await this.network.sendBeaconBlocksByRoot(peer, [anchorBlockRoot]); - if (anchorBlock == null) throw new Error("InvalidBlockSyncedFromPeer"); + const res = await this.network.sendBeaconBlocksByRoot(peer, [anchorBlockRoot]); + if (res.length < 1) throw new Error("InvalidBlockSyncedFromPeer"); + const anchorBlock = res[0]; // GENESIS_SLOT doesn't has valid signature - if (anchorBlock.message.slot === GENESIS_SLOT) return; + if (anchorBlock.data.message.slot === GENESIS_SLOT) return; await verifyBlockProposerSignature(this.chain.bls, this.chain.getHeadState(), [anchorBlock]); // We can write to the disk if this is ahead of prevFinalizedCheckpointBlock otherwise // we will need to go make checks on the top of sync loop before writing as it might // override prevFinalizedCheckpointBlock - if (this.prevFinalizedCheckpointBlock.slot < anchorBlock.message.slot) - await this.db.blockArchive.put(anchorBlock.message.slot, anchorBlock); + if (this.prevFinalizedCheckpointBlock.slot < anchorBlock.data.message.slot) + await this.db.blockArchive.putBinary(anchorBlock.data.message.slot, anchorBlock.bytes); this.syncAnchor = { - anchorBlock, + anchorBlock: anchorBlock.data, anchorBlockRoot, - anchorSlot: anchorBlock.message.slot, - lastBackSyncedBlock: {root: anchorBlockRoot, slot: anchorBlock.message.slot, block: anchorBlock}, + anchorSlot: anchorBlock.data.message.slot, + lastBackSyncedBlock: {root: anchorBlockRoot, slot: anchorBlock.data.message.slot, block: anchorBlock.data}, }; this.metrics?.backfillSync.totalBlocks.inc({method: BackfillSyncMethod.blockbyroot}); this.logger.verbose("Fetched new anchorBlock", { root: toHexString(anchorBlockRoot), - slot: anchorBlock.message.slot, + slot: anchorBlock.data.message.slot, }); return; @@ -822,10 +823,18 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} // Verified blocks are in reverse order with the nextAnchor being the smallest slot // if nextAnchor is on the same slot as prevFinalizedCheckpointBlock, we can't save // it before returning to top of sync loop for validation - await this.db.blockArchive.batchAdd( + const blocksToPut = nextAnchor.slot > this.prevFinalizedCheckpointBlock.slot ? verifiedBlocks - : verifiedBlocks.slice(0, verifiedBlocks.length - 1) + : verifiedBlocks.slice(0, verifiedBlocks.length - 1); + await this.db.blockArchive.batchPutBinary( + blocksToPut.map((block) => ({ + key: block.data.message.slot, + value: block.bytes, + slot: block.data.message.slot, + blockRoot: this.config.getForkTypes(block.data.message.slot).BeaconBlock.hashTreeRoot(block.data.message), + parentRoot: block.data.message.parentRoot, + })) ); this.metrics?.backfillSync.totalBlocks.inc({method: BackfillSyncMethod.rangesync}, verifiedBlocks.length); } diff --git a/packages/beacon-node/src/sync/backfill/verify.ts b/packages/beacon-node/src/sync/backfill/verify.ts index 36a00bea6833..eba4feca48ef 100644 --- a/packages/beacon-node/src/sync/backfill/verify.ts +++ b/packages/beacon-node/src/sync/backfill/verify.ts @@ -3,6 +3,7 @@ import {BeaconConfig} from "@lodestar/config"; import {allForks, Root, allForks as allForkTypes, ssz, Slot} from "@lodestar/types"; import {GENESIS_SLOT} from "@lodestar/params"; import {IBlsVerifier} from "../../chain/bls/index.js"; +import {WithBytes} from "../../network/interface.js"; import {BackfillSyncError, BackfillSyncErrorCode} from "./errors.js"; export type BackfillBlockHeader = { @@ -14,19 +15,19 @@ export type BackfillBlock = BackfillBlockHeader & {block: allForks.SignedBeaconB export function verifyBlockSequence( config: BeaconConfig, - blocks: allForkTypes.SignedBeaconBlock[], + blocks: WithBytes[], anchorRoot: Root ): { nextAnchor: BackfillBlock | null; - verifiedBlocks: allForkTypes.SignedBeaconBlock[]; + verifiedBlocks: WithBytes[]; error?: BackfillSyncErrorCode.NOT_LINEAR; } { let nextRoot: Root = anchorRoot; let nextAnchor: BackfillBlock | null = null; - const verifiedBlocks: allForkTypes.SignedBeaconBlock[] = []; + const verifiedBlocks: WithBytes[] = []; for (const block of blocks.reverse()) { - const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); + const blockRoot = config.getForkTypes(block.data.message.slot).BeaconBlock.hashTreeRoot(block.data.message); if (!ssz.Root.equals(blockRoot, nextRoot)) { if (ssz.Root.equals(nextRoot, anchorRoot)) { throw new BackfillSyncError({code: BackfillSyncErrorCode.NOT_ANCHORED}); @@ -34,8 +35,8 @@ export function verifyBlockSequence( return {nextAnchor, verifiedBlocks, error: BackfillSyncErrorCode.NOT_LINEAR}; } verifiedBlocks.push(block); - nextAnchor = {block, slot: block.message.slot, root: nextRoot}; - nextRoot = block.message.parentRoot; + nextAnchor = {block: block.data, slot: block.data.message.slot, root: nextRoot}; + nextRoot = block.data.message.parentRoot; } return {nextAnchor, verifiedBlocks}; } @@ -43,12 +44,12 @@ export function verifyBlockSequence( export async function verifyBlockProposerSignature( bls: IBlsVerifier, state: CachedBeaconStateAllForks, - blocks: allForkTypes.SignedBeaconBlock[] + blocks: WithBytes[] ): Promise { - if (blocks.length === 1 && blocks[0].message.slot === GENESIS_SLOT) return; + if (blocks.length === 1 && blocks[0].data.message.slot === GENESIS_SLOT) return; const signatures = blocks.reduce((sigs: ISignatureSet[], block) => { // genesis block doesn't have valid signature - if (block.message.slot !== GENESIS_SLOT) sigs.push(getBlockProposerSignatureSet(state, block)); + if (block.data.message.slot !== GENESIS_SLOT) sigs.push(getBlockProposerSignatureSet(state, block.data)); return sigs; }, []); diff --git a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts index a8802d17544e..552f1cfd5ae2 100644 --- a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts +++ b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts @@ -84,6 +84,7 @@ describe("data serialization through worker boundary", function () { type: BlockInputType.preDeneb, block: ssz.capella.SignedBeaconBlock.defaultValue(), source: BlockSource.gossip, + blockBytes: ZERO_HASH, }, peer, }, diff --git a/packages/beacon-node/test/e2e/network/reqresp.test.ts b/packages/beacon-node/test/e2e/network/reqresp.test.ts index a46918b62813..1a334350a08a 100644 --- a/packages/beacon-node/test/e2e/network/reqresp.test.ts +++ b/packages/beacon-node/test/e2e/network/reqresp.test.ts @@ -144,7 +144,10 @@ function runTests(this: Mocha.Suite, {useWorker}: {useWorker: boolean}): void { expect(returnedBlocks).to.have.length(req.count, "Wrong returnedBlocks length"); for (const [i, returnedBlock] of returnedBlocks.entries()) { - expect(ssz.phase0.SignedBeaconBlock.equals(returnedBlock, blocks[i])).to.equal(true, `Wrong returnedBlock[${i}]`); + expect(ssz.phase0.SignedBeaconBlock.equals(returnedBlock.data, blocks[i])).to.equal( + true, + `Wrong returnedBlock[${i}]` + ); } }); diff --git a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts index da045b15738a..4a4a5e4e5d5b 100644 --- a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts @@ -117,7 +117,7 @@ describe("sync / unknown block sync", function () { ); await connect(bn2.network, bn.network); - const headInput = getBlockInput.preDeneb(config, head, BlockSource.gossip); + const headInput = getBlockInput.preDeneb(config, head, BlockSource.gossip, null); switch (event) { case NetworkEvent.unknownBlockParent: diff --git a/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts b/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts index c819afa4ba68..862d6d594b15 100644 --- a/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts +++ b/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts @@ -107,7 +107,7 @@ describe.skip("verify+import blocks - range sync perf test", () => { }, fn: async (chain) => { const blocksImport = blocks.value.map((block) => - getBlockInput.preDeneb(chain.config, block, BlockSource.byRange) + getBlockInput.preDeneb(chain.config, block, BlockSource.byRange, null) ); await chain.processChainSegment(blocksImport, { diff --git a/packages/beacon-node/test/spec/presets/fork_choice.ts b/packages/beacon-node/test/spec/presets/fork_choice.ts index 27a5e8f0a172..3ebe850c75e2 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.ts @@ -157,12 +157,13 @@ export const forkChoiceTest = const blockImport = config.getForkSeq(slot) < ForkSeq.deneb - ? getBlockInput.preDeneb(config, signedBlock, BlockSource.gossip) + ? getBlockInput.preDeneb(config, signedBlock, BlockSource.gossip, null) : getBlockInput.postDeneb( config, signedBlock, BlockSource.gossip, - getEmptyBlobsSidecar(config, signedBlock as deneb.SignedBeaconBlock) + getEmptyBlobsSidecar(config, signedBlock as deneb.SignedBeaconBlock), + null ); try { diff --git a/packages/beacon-node/test/unit/chain/blocks/verifyBlocksSanityChecks.test.ts b/packages/beacon-node/test/unit/chain/blocks/verifyBlocksSanityChecks.test.ts index 88590a6ff495..5b7ac2fe6062 100644 --- a/packages/beacon-node/test/unit/chain/blocks/verifyBlocksSanityChecks.test.ts +++ b/packages/beacon-node/test/unit/chain/blocks/verifyBlocksSanityChecks.test.ts @@ -127,7 +127,7 @@ function verifyBlocksSanityChecks( ): {relevantBlocks: allForks.SignedBeaconBlock[]; parentSlots: Slot[]; parentBlock: ProtoBlock | null} { const {relevantBlocks, parentSlots, parentBlock} = verifyBlocksImportSanityChecks( modules, - blocks.map((block) => getBlockInput.preDeneb(config, block, BlockSource.byRange)), + blocks.map((block) => getBlockInput.preDeneb(config, block, BlockSource.byRange, null)), opts ); return { diff --git a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts index df85d4b63165..37e5afa4d025 100644 --- a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts +++ b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts @@ -4,9 +4,10 @@ import {createBeaconConfig, createChainForkConfig, defaultChainConfig} from "@lo import {BYTES_PER_FIELD_ELEMENT} from "@lodestar/params"; import {beaconBlocksMaybeBlobsByRange} from "../../../src/network/reqresp/index.js"; -import {BlockInputType, BlockSource, blobSidecarsToBlobsSidecar} from "../../../src/chain/blocks/types.js"; +import {BlockInput, BlockInputType, BlockSource, blobSidecarsToBlobsSidecar} from "../../../src/chain/blocks/types.js"; import {ckzg, initCKZG, loadEthereumTrustedSetup, FIELD_ELEMENTS_PER_BLOB_MAINNET} from "../../../src/util/kzg.js"; import {INetwork} from "../../../src/network/interface.js"; +import {ZERO_HASH} from "../../../src/constants/constants.js"; describe("beaconBlocksMaybeBlobsByRange", () => { before(async function () { @@ -88,7 +89,7 @@ describe("beaconBlocksMaybeBlobsByRange", () => { .filter((blobs) => blobs !== undefined) .reduce((acc, elem) => acc.concat(elem), []); - const expectedResponse = blocksWithBlobs.map(([block, blobSidecars]) => { + const expectedResponse: BlockInput[] = blocksWithBlobs.map(([block, blobSidecars]) => { const blobs = (blobSidecars !== undefined ? blobSidecars : []).map((bscar) => { // TODO DENEB: cleanup the following generation as its not required to generate // proper field elements for the aggregate proofs compute @@ -103,11 +104,16 @@ describe("beaconBlocksMaybeBlobsByRange", () => { source: BlockSource.byRange, // TODO DENEB: Cleanup the conversion once migration complete blobs: blobSidecarsToBlobsSidecar(chainConfig, block, blobs), + blockBytes: null, }; }); const network = { - sendBeaconBlocksByRange: async () => blocks, + sendBeaconBlocksByRange: async () => + blocks.map((data) => ({ + data, + bytes: ZERO_HASH, + })), sendBlobSidecarsByRange: async () => blobSidecars, } as Partial as INetwork; diff --git a/packages/beacon-node/test/unit/sync/backfill/verify.test.ts b/packages/beacon-node/test/unit/sync/backfill/verify.test.ts index 8e106a179323..18ce67fab40b 100644 --- a/packages/beacon-node/test/unit/sync/backfill/verify.test.ts +++ b/packages/beacon-node/test/unit/sync/backfill/verify.test.ts @@ -6,6 +6,8 @@ import {createBeaconConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; import {phase0, ssz} from "@lodestar/types"; import {verifyBlockSequence} from "../../../../src/sync/backfill/verify.js"; +import {WithBytes} from "../../../../src/network/interface.js"; +import {ZERO_HASH} from "../../../../src/constants/constants.js"; import {BackfillSyncErrorCode, BackfillSyncError} from "./../../../../src/sync/backfill/errors.js"; // Global variable __dirname no longer available in ES6 modules. @@ -23,7 +25,7 @@ describe("backfill sync - verify block sequence", function () { it("should verify valid chain of blocks", function () { const blocks = getBlocks(); - expect(() => verifyBlockSequence(beaconConfig, blocks.slice(0, 2), blocks[2].message.parentRoot)).to.not.throw; + expect(() => verifyBlockSequence(beaconConfig, blocks.slice(0, 2), blocks[2].data.message.parentRoot)).to.not.throw; }); it("should fail with sequence not anchored", function () { @@ -41,18 +43,18 @@ describe("backfill sync - verify block sequence", function () { const {error} = verifyBlockSequence( beaconConfig, // remove middle block - blocks.filter((b) => b.message.slot !== 2).slice(0, blocks.length - 2), - blocks[blocks.length - 1].message.parentRoot + blocks.filter((b) => b.data.message.slot !== 2).slice(0, blocks.length - 2), + blocks[blocks.length - 1].data.message.parentRoot ); if (error) throw new BackfillSyncError({code: error}); }).to.throw(BackfillSyncErrorCode.NOT_LINEAR); }); //first 4 mainnet blocks - function getBlocks(): phase0.SignedBeaconBlock[] { + function getBlocks(): WithBytes[] { const json = JSON.parse(fs.readFileSync(path.join(__dirname, "./blocks.json"), "utf-8")) as unknown[]; return json.map((b) => { - return ssz.phase0.SignedBeaconBlock.fromJson(b); + return {data: ssz.phase0.SignedBeaconBlock.fromJson(b), bytes: ZERO_HASH}; }); } }); diff --git a/packages/beacon-node/test/unit/sync/range/batch.test.ts b/packages/beacon-node/test/unit/sync/range/batch.test.ts index 78e9345dda73..51ca12b72ffa 100644 --- a/packages/beacon-node/test/unit/sync/range/batch.test.ts +++ b/packages/beacon-node/test/unit/sync/range/batch.test.ts @@ -12,7 +12,7 @@ describe("sync / range / batch", async () => { const startEpoch = 0; const peer = validPeerIdStr; const blocksDownloaded = [ - getBlockInput.preDeneb(config, ssz.phase0.SignedBeaconBlock.defaultValue(), BlockSource.byRange), + getBlockInput.preDeneb(config, ssz.phase0.SignedBeaconBlock.defaultValue(), BlockSource.byRange, null), ]; it("Should return correct blockByRangeRequest", () => { diff --git a/packages/beacon-node/test/unit/sync/range/chain.test.ts b/packages/beacon-node/test/unit/sync/range/chain.test.ts index a2e09f784056..ecaa8a0105fc 100644 --- a/packages/beacon-node/test/unit/sync/range/chain.test.ts +++ b/packages/beacon-node/test/unit/sync/range/chain.test.ts @@ -89,7 +89,8 @@ describe("sync / range / chain", () => { message: generateEmptyBlock(i), signature: shouldReject ? REJECT_BLOCK : ACCEPT_BLOCK, }, - BlockSource.byRange + BlockSource.byRange, + null ) ); } @@ -134,7 +135,8 @@ describe("sync / range / chain", () => { message: generateEmptyBlock(i), signature: ACCEPT_BLOCK, }, - BlockSource.byRange + BlockSource.byRange, + null ) ); } diff --git a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts index 6086ee732858..0e942e5b8ce0 100644 --- a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts +++ b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts @@ -16,6 +16,7 @@ import {ClockStopped} from "../../utils/mocks/clock.js"; import {SeenBlockProposers} from "../../../src/chain/seenCache/seenBlockProposers.js"; import {BlockError, BlockErrorCode} from "../../../src/chain/errors/blockError.js"; import {defaultSyncOptions} from "../../../src/sync/options.js"; +import {ZERO_HASH} from "../../../src/constants/constants.js"; describe("sync / UnknownBlockSync", () => { const logger = testLogger(); @@ -130,8 +131,11 @@ describe("sync / UnknownBlockSync", () => { sendBeaconBlocksByRootResolveFn([_peerId, roots]); const correctBlocks = Array.from(roots) .map((root) => blocksByRoot.get(toHexString(root))) - .filter(notNullish); - return wrongBlockRoot ? [ssz.phase0.SignedBeaconBlock.defaultValue()] : correctBlocks; + .filter(notNullish) + .map((data) => ({data, bytes: ZERO_HASH})); + return wrongBlockRoot + ? [{data: ssz.phase0.SignedBeaconBlock.defaultValue(), bytes: ZERO_HASH}] + : correctBlocks; }, reportPeer: async (peerId, action, actionName) => reportPeerResolveFn([peerId, action, actionName]), @@ -180,7 +184,7 @@ describe("sync / UnknownBlockSync", () => { syncService.subscribeToNetwork(); if (event === NetworkEvent.unknownBlockParent) { network.events?.emit(NetworkEvent.unknownBlockParent, { - blockInput: getBlockInput.preDeneb(config, blockC, BlockSource.gossip), + blockInput: getBlockInput.preDeneb(config, blockC, BlockSource.gossip, null), peer, }); } else { diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index d865c7cf9bc1..e2e416fa3667 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -20,5 +20,3 @@ export enum ProducedBlockSource { export type SlotRootHex = {slot: Slot; root: RootHex}; export type SlotOptionalRoot = {slot: Slot; root?: RootHex}; -export type WithBytes> = T & {serializedData: Uint8Array}; -export type WithOptionalBytes> = T & {serializedData?: Uint8Array}; From 7c54a7b188e8bb7f15a2b9d742da5e5dfc887c1d Mon Sep 17 00:00:00 2001 From: g11tech Date: Mon, 19 Jun 2023 20:37:42 +0530 Subject: [PATCH 06/96] feat: allow builder only proposal flow for benefit of DVT protocols (#5669) * feat: allow builder only proposal flow for benefit of DVT protocols * fix typo * apply feedback * update descp * add suggestion Co-authored-by: Nico Flaig --------- Co-authored-by: Nico Flaig --- packages/cli/src/cmds/validator/handler.ts | 2 + packages/cli/src/cmds/validator/options.ts | 2 +- packages/validator/src/services/block.ts | 39 +++++++++++++++---- .../validator/src/services/validatorStore.ts | 2 + 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index 305e101f5287..c0f8e6babdc2 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -254,6 +254,8 @@ function parseBuilderSelection(builderSelection?: string): BuilderSelection | un break; case "builderalways": break; + case "builderonly": + break; default: throw Error("Invalid input for builder selection, check help."); } diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index 3dd0d7d9505d..f92814cdcb3d 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -237,7 +237,7 @@ export const validatorOptions: CliCommandOptions = { "builder.selection": { type: "string", - description: "Default builder block selection strategy: maxprofit or builderalways", + description: "Default builder block selection strategy: maxprofit, builderalways, or builderonly", defaultDescription: `${defaultOptions.builderSelection}`, group: "builder", }, diff --git a/packages/validator/src/services/block.ts b/packages/validator/src/services/block.ts index bb2d230e0321..b83c239b62e8 100644 --- a/packages/validator/src/services/block.ts +++ b/packages/validator/src/services/block.ts @@ -109,14 +109,20 @@ export class BlockProposingService { const debugLogCtx = {...logCtx, validator: pubkeyHex}; - this.logger.debug("Producing block", debugLogCtx); - this.metrics?.proposerStepCallProduceBlock.observe(this.clock.secFromSlot(slot)); - const strictFeeRecipientCheck = this.validatorStore.strictFeeRecipientCheck(pubkeyHex); const isBuilderEnabled = this.validatorStore.isBuilderEnabled(pubkeyHex); const builderSelection = this.validatorStore.getBuilderSelection(pubkeyHex); const expectedFeeRecipient = this.validatorStore.getFeeRecipient(pubkeyHex); + this.logger.debug("Producing block", { + ...debugLogCtx, + isBuilderEnabled, + builderSelection, + expectedFeeRecipient, + strictFeeRecipientCheck, + }); + this.metrics?.proposerStepCallProduceBlock.observe(this.clock.secFromSlot(slot)); + const blockContents = await this.produceBlockWrapper(slot, randaoReveal, graffiti, { expectedFeeRecipient, strictFeeRecipientCheck, @@ -190,10 +196,20 @@ export class BlockProposingService { > => { // Start calls for building execution and builder blocks const blindedBlockPromise = isBuilderEnabled ? this.produceBlindedBlock(slot, randaoReveal, graffiti) : null; - const fullBlockPromise = this.produceBlock(slot, randaoReveal, graffiti); + const fullBlockPromise = + // At any point either the builder or execution or both flows should be active. + // + // Ideally such a scenario should be prevented on startup, but proposerSettingsFile or keymanager + // configurations could cause a validator pubkey to have builder disabled with builder selection builder only + // (TODO: independently make sure such an options update is not successful for a validator pubkey) + // + // So if builder is disabled ignore builder selection of builderonly if caused by user mistake + !isBuilderEnabled || builderSelection !== BuilderSelection.BuilderOnly + ? this.produceBlock(slot, randaoReveal, graffiti) + : null; let blindedBlock, fullBlock; - if (blindedBlockPromise !== null) { + if (blindedBlockPromise !== null && fullBlockPromise !== null) { // reference index of promises in the race const promisesOrder = [ProducedBlockSource.builder, ProducedBlockSource.engine]; [blindedBlock, fullBlock] = await racePromisesWithCutoff<{ @@ -225,9 +241,16 @@ export class BlockProposingService { this.logger.error("Failed to produce execution block", {}, fullBlock); fullBlock = null; } - } else { - fullBlock = await fullBlockPromise; + } else if (blindedBlockPromise !== null && fullBlockPromise === null) { + blindedBlock = await blindedBlockPromise; + fullBlock = null; + } else if (blindedBlockPromise === null && fullBlockPromise !== null) { blindedBlock = null; + fullBlock = await fullBlockPromise; + } else { + throw Error( + `Internal Error: Neither builder nor execution proposal flow activated isBuilderEnabled=${isBuilderEnabled} builderSelection=${builderSelection}` + ); } const builderBlockValue = blindedBlock?.blockValue ?? BigInt(0); @@ -252,7 +275,7 @@ export class BlockProposingService { break; } - case BuilderSelection.BuilderAlways: + // For everything else just select the builder default: { selectedSource = ProducedBlockSource.builder; selectedBlock = blindedBlock; diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 0dbd9077257b..d2fe36b1e214 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -67,6 +67,8 @@ export type SignerRemote = { export enum BuilderSelection { BuilderAlways = "builderalways", MaxProfit = "maxprofit", + /** Only activate builder flow for DVT block proposal protocols */ + BuilderOnly = "builderonly", } type DefaultProposerConfig = { From e60af6465f04bb6abdd44bce8fede4feb2e9f7df Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Tue, 20 Jun 2023 16:37:03 +0700 Subject: [PATCH 07/96] chore: track block source in lodestar summary dashboard (#5668) * chore: track block source in lodestar summary dashboard * Add job vars to __inputs --------- Co-authored-by: Nico Flaig --- dashboards/lodestar_summary.json | 279 ++++++++++++++++++++++++------- 1 file changed, 223 insertions(+), 56 deletions(-) diff --git a/dashboards/lodestar_summary.json b/dashboards/lodestar_summary.json index d3c93ab64d39..349fab48a81a 100644 --- a/dashboards/lodestar_summary.json +++ b/dashboards/lodestar_summary.json @@ -1,12 +1,12 @@ { "__inputs": [ { - "name": "DS_PROMETHEUS", - "label": "Prometheus", "description": "", - "type": "datasource", + "label": "Prometheus", + "name": "DS_PROMETHEUS", "pluginId": "prometheus", - "pluginName": "Prometheus" + "pluginName": "Prometheus", + "type": "datasource" }, { "name": "VAR_BEACON_JOB", @@ -1743,7 +1743,7 @@ } ] }, - "unit": "s" + "unit": "percentunit" }, "overrides": [] }, @@ -1753,6 +1753,191 @@ "x": 12, "y": 22 }, + "id": 536, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_gossip_block_elapsed_time_till_processed_count[$rate_interval])\n- on(instance)\nrate(lodestar_gossip_block_elapsed_time_till_processed_bucket{le=\"4\"}[$rate_interval])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Import head late (> 4 sec) rate ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 538, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_import_block_by_source_total[$rate_interval]) * 12", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Source", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 30 + }, "id": 532, "options": { "legend": { @@ -1840,7 +2025,7 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 38 }, "id": 487, "panels": [], @@ -1887,7 +2072,7 @@ "h": 3, "w": 4, "x": 0, - "y": 31 + "y": 39 }, "id": 493, "options": { @@ -1952,7 +2137,7 @@ "h": 3, "w": 4, "x": 4, - "y": 31 + "y": 39 }, "id": 492, "options": { @@ -2017,7 +2202,7 @@ "h": 3, "w": 4, "x": 8, - "y": 31 + "y": 39 }, "id": 484, "options": { @@ -2082,7 +2267,7 @@ "h": 3, "w": 4, "x": 12, - "y": 31 + "y": 39 }, "id": 483, "options": { @@ -2147,7 +2332,7 @@ "h": 3, "w": 4, "x": 16, - "y": 31 + "y": 39 }, "id": 485, "options": { @@ -2225,7 +2410,7 @@ "h": 3, "w": 4, "x": 20, - "y": 31 + "y": 39 }, "id": 488, "options": { @@ -2320,7 +2505,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 42 }, "id": 490, "options": { @@ -2384,7 +2569,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 42 }, "heatmap": {}, "hideZeroBuckets": false, @@ -2496,7 +2681,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 50 }, "heatmap": {}, "hideZeroBuckets": false, @@ -2616,8 +2801,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2632,7 +2816,7 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 50 }, "id": 497, "options": { @@ -2660,7 +2844,7 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 58 }, "id": 104, "panels": [], @@ -2722,8 +2906,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2739,7 +2922,7 @@ "h": 6, "w": 12, "x": 0, - "y": 51 + "y": 59 }, "id": 102, "options": { @@ -2844,8 +3027,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2861,7 +3043,7 @@ "h": 6, "w": 12, "x": 12, - "y": 51 + "y": 59 }, "id": 172, "options": { @@ -2942,8 +3124,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2959,7 +3140,7 @@ "h": 6, "w": 12, "x": 0, - "y": 57 + "y": 65 }, "id": 171, "options": { @@ -3002,7 +3183,7 @@ "h": 6, "w": 12, "x": 12, - "y": 57 + "y": 65 }, "id": 99, "options": { @@ -3156,46 +3337,32 @@ "type": "adhoc" }, { + "current": { + "selected": false, + "text": "${VAR_BEACON_JOB}", + "value": "${VAR_BEACON_JOB}" + }, "description": "Job name used in Prometheus config to scrape Beacon node", "hide": 2, "label": "Beacon node job name", "name": "beacon_job", "query": "${VAR_BEACON_JOB}", "skipUrlSync": false, - "type": "constant", - "current": { - "value": "${VAR_BEACON_JOB}", - "text": "${VAR_BEACON_JOB}", - "selected": false - }, - "options": [ - { - "value": "${VAR_BEACON_JOB}", - "text": "${VAR_BEACON_JOB}", - "selected": false - } - ] + "type": "constant" }, { + "current": { + "selected": false, + "text": "${VAR_VALIDATOR_JOB}", + "value": "${VAR_VALIDATOR_JOB}" + }, "description": "Job name used in Prometheus config to scrape Validator client", "hide": 2, "label": "Validator client job name", "name": "validator_job", "query": "${VAR_VALIDATOR_JOB}", "skipUrlSync": false, - "type": "constant", - "current": { - "value": "${VAR_VALIDATOR_JOB}", - "text": "${VAR_VALIDATOR_JOB}", - "selected": false - }, - "options": [ - { - "value": "${VAR_VALIDATOR_JOB}", - "text": "${VAR_VALIDATOR_JOB}", - "selected": false - } - ] + "type": "constant" } ] }, @@ -3219,7 +3386,7 @@ }, "timezone": "utc", "title": "Lodestar", - "uid": "lodestar", + "uid": "lodestar_summary", "version": 29, "weekStart": "monday" } From d1ba8c1473296d4f09f2c4ae42440a56e3dc30c1 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 20 Jun 2023 11:56:40 +0200 Subject: [PATCH 08/96] chore: add job names to __inputs if used by dashboard (#5672) * Add job names to __inputs if used by dashboard * Reorder properties in prometheus input --- scripts/lint-grafana-dashboard.mjs | 47 +++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/scripts/lint-grafana-dashboard.mjs b/scripts/lint-grafana-dashboard.mjs index 457a4db6570b..70b93fe0df0c 100644 --- a/scripts/lint-grafana-dashboard.mjs +++ b/scripts/lint-grafana-dashboard.mjs @@ -71,6 +71,7 @@ no-console * * @typedef {Object} TemplatingListItem * @property {string} name + * @property {string} query */ const variableNameDatasource = "DS_PROMETHEUS"; @@ -86,18 +87,44 @@ export function lintGrafanaDashboard(json) { delete json.__elements; delete json.__requires; + // Always add Prometheus to __inputs + const inputs = [ + { + name: variableNameDatasource, + type: "datasource", + label: "Prometheus", + description: "", + pluginId: "prometheus", + pluginName: "Prometheus", + }, + ]; + + // Add job names to __inputs if used by dashboard + if (json.templating && json.templating.list) { + for (const item of json.templating.list) { + if (item.query === "${VAR_BEACON_JOB}") { + inputs.push({ + name: "VAR_BEACON_JOB", + type: "constant", + label: "Beacon node job name", + value: "beacon", + description: "", + }); + } else if (item.query === "${VAR_VALIDATOR_JOB}") { + inputs.push({ + name: "VAR_VALIDATOR_JOB", + type: "constant", + label: "Validator client job name", + value: "validator", + description: "", + }); + } + } + } + // Ensure __inputs is first property to reduce diff json = { - __inputs: [ - { - description: "", - label: "Prometheus", - name: variableNameDatasource, - pluginId: "prometheus", - pluginName: "Prometheus", - type: "datasource", - }, - ], + __inputs: inputs, ...json, }; From e8b53ea874ad42213295db43f21b67341789b736 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 20 Jun 2023 12:57:31 +0300 Subject: [PATCH 09/96] chore: typesafe options and cli args (#4576) * Typesafe CLI args * Set execution mode to "http" in default options * Fix type of default retryAttempts and retryDelay * Unconditionally use default execution http opts in CLI args --------- Co-authored-by: Nico Flaig --- .../beacon-node/src/eth1/eth1DepositsCache.ts | 4 +- packages/beacon-node/src/eth1/options.ts | 10 +++-- .../src/eth1/provider/eth1Provider.ts | 4 +- .../beacon-node/src/monitoring/options.ts | 5 +-- .../beacon-node/src/monitoring/service.ts | 2 +- packages/beacon-node/src/network/options.ts | 6 +-- .../src/network/peers/peerManager.ts | 6 ++- .../src/network/processor/gossipHandlers.ts | 11 +---- packages/beacon-node/src/node/nodejs.ts | 9 ++-- packages/beacon-node/src/node/options.ts | 5 +++ packages/cli/src/cmds/beacon/options.ts | 19 ++++----- packages/cli/src/cmds/lightclient/handler.ts | 5 +++ packages/cli/src/cmds/lightclient/options.ts | 6 +-- .../cmds/validator/blsToExecutionChange.ts | 22 ++++++---- packages/cli/src/cmds/validator/options.ts | 4 +- .../validator/slashingProtection/export.ts | 9 ++-- .../validator/slashingProtection/import.ts | 11 +++-- .../cli/src/options/beaconNodeOptions/api.ts | 16 +++---- .../src/options/beaconNodeOptions/builder.ts | 22 ++++------ .../src/options/beaconNodeOptions/chain.ts | 31 +++++++------- .../cli/src/options/beaconNodeOptions/eth1.ts | 14 +++---- .../options/beaconNodeOptions/execution.ts | 16 +++---- .../src/options/beaconNodeOptions/metrics.ts | 6 +-- .../options/beaconNodeOptions/monitoring.ts | 10 ++--- .../src/options/beaconNodeOptions/network.ts | 42 +++++++++---------- .../cli/src/options/beaconNodeOptions/sync.ts | 10 ++--- packages/cli/src/options/globalOptions.ts | 2 +- packages/cli/src/util/command.ts | 7 +++- 28 files changed, 160 insertions(+), 154 deletions(-) diff --git a/packages/beacon-node/src/eth1/eth1DepositsCache.ts b/packages/beacon-node/src/eth1/eth1DepositsCache.ts index c907b1a5b7da..b2c3c0c83a11 100644 --- a/packages/beacon-node/src/eth1/eth1DepositsCache.ts +++ b/packages/beacon-node/src/eth1/eth1DepositsCache.ts @@ -15,10 +15,10 @@ export class Eth1DepositsCache { db: IBeaconDb; config: ChainForkConfig; - constructor(opts: {unsafeAllowDepositDataOverwrite: boolean}, config: ChainForkConfig, db: IBeaconDb) { + constructor(opts: {unsafeAllowDepositDataOverwrite?: boolean}, config: ChainForkConfig, db: IBeaconDb) { this.config = config; this.db = db; - this.unsafeAllowDepositDataOverwrite = opts.unsafeAllowDepositDataOverwrite; + this.unsafeAllowDepositDataOverwrite = opts.unsafeAllowDepositDataOverwrite ?? false; } /** diff --git a/packages/beacon-node/src/eth1/options.ts b/packages/beacon-node/src/eth1/options.ts index 393c2310af9d..0f49f2d8a95c 100644 --- a/packages/beacon-node/src/eth1/options.ts +++ b/packages/beacon-node/src/eth1/options.ts @@ -1,14 +1,14 @@ export type Eth1Options = { - enabled: boolean; + enabled?: boolean; disableEth1DepositDataTracker?: boolean; - providerUrls: string[]; + providerUrls?: string[]; /** * jwtSecretHex is the jwt secret if the eth1 modules should ping the jwt auth * protected engine endpoints. */ jwtSecretHex?: string; depositContractDeployBlock?: number; - unsafeAllowDepositDataOverwrite: boolean; + unsafeAllowDepositDataOverwrite?: boolean; /** * Vote for a specific eth1_data regardless of validity and existing votes. * hex encoded ssz serialized Eth1Data type. @@ -16,9 +16,11 @@ export type Eth1Options = { forcedEth1DataVote?: string; }; +export const DEFAULT_PROVIDER_URLS = ["http://localhost:8545"]; + export const defaultEth1Options: Eth1Options = { enabled: true, - providerUrls: ["http://localhost:8545"], + providerUrls: DEFAULT_PROVIDER_URLS, depositContractDeployBlock: 0, unsafeAllowDepositDataOverwrite: false, }; diff --git a/packages/beacon-node/src/eth1/provider/eth1Provider.ts b/packages/beacon-node/src/eth1/provider/eth1Provider.ts index b44b29bc69f3..3fe5913d7a87 100644 --- a/packages/beacon-node/src/eth1/provider/eth1Provider.ts +++ b/packages/beacon-node/src/eth1/provider/eth1Provider.ts @@ -6,7 +6,7 @@ import {fromHex} from "@lodestar/utils"; import {linspace} from "../../util/numpy.js"; import {depositEventTopics, parseDepositLog} from "../utils/depositContract.js"; import {Eth1Block, IEth1Provider} from "../interface.js"; -import {Eth1Options} from "../options.js"; +import {DEFAULT_PROVIDER_URLS, Eth1Options} from "../options.js"; import {isValidAddress} from "../../util/address.js"; import {EthJsonRpcBlockRaw} from "../interface.js"; import {JsonRpcHttpClient, JsonRpcHttpClientMetrics, ReqOpts} from "./jsonRpcHttpClient.js"; @@ -55,7 +55,7 @@ export class Eth1Provider implements IEth1Provider { ) { this.deployBlock = opts.depositContractDeployBlock ?? 0; this.depositContractAddress = toHexString(config.DEPOSIT_CONTRACT_ADDRESS); - this.rpc = new JsonRpcHttpClient(opts.providerUrls, { + this.rpc = new JsonRpcHttpClient(opts.providerUrls ?? DEFAULT_PROVIDER_URLS, { signal, // Don't fallback with is truncated error. Throw early and let the retry on this class handle it shouldNotFallback: isJsonRpcTruncatedError, diff --git a/packages/beacon-node/src/monitoring/options.ts b/packages/beacon-node/src/monitoring/options.ts index a9ab2cd8172e..22d8f69ed629 100644 --- a/packages/beacon-node/src/monitoring/options.ts +++ b/packages/beacon-node/src/monitoring/options.ts @@ -1,6 +1,6 @@ export type MonitoringOptions = { /** Remote endpoint URL where client stats are sent */ - endpoint: string; + endpoint?: string; /** Interval in milliseconds between sending client stats */ interval?: number; /** Initial delay in milliseconds before client stats are sent */ @@ -11,8 +11,7 @@ export type MonitoringOptions = { collectSystemStats?: boolean; }; -export const defaultMonitoringOptions: Required = { - endpoint: "", +export const defaultMonitoringOptions: Required> = { interval: 60_000, initialDelay: 30_000, requestTimeout: 10_000, diff --git a/packages/beacon-node/src/monitoring/service.ts b/packages/beacon-node/src/monitoring/service.ts index 33938b9281dc..7f8f740e46f6 100644 --- a/packages/beacon-node/src/monitoring/service.ts +++ b/packages/beacon-node/src/monitoring/service.ts @@ -49,7 +49,7 @@ export class MonitoringService { constructor( client: Client, - options: MonitoringOptions, + options: Required> & MonitoringOptions, {register, logger}: {register: RegistryMetricCreator; logger: Logger} ) { this.options = {...defaultMonitoringOptions, ...options}; diff --git a/packages/beacon-node/src/network/options.ts b/packages/beacon-node/src/network/options.ts index 7d0b73b62bf8..095d2b0bc3d1 100644 --- a/packages/beacon-node/src/network/options.ts +++ b/packages/beacon-node/src/network/options.ts @@ -1,5 +1,4 @@ import {Eth2GossipsubOpts} from "./gossip/gossipsub.js"; -import {defaultGossipHandlerOpts} from "./processor/gossipHandlers.js"; import {PeerManagerOpts, PeerRpcScoreOpts} from "./peers/index.js"; import {ReqRespBeaconNodeOpts} from "./reqresp/ReqRespBeaconNode.js"; import {NetworkProcessorOpts} from "./processor/index.js"; @@ -15,7 +14,7 @@ export interface NetworkOptions localMultiaddrs: string[]; bootMultiaddrs?: string[]; subscribeAllSubnets?: boolean; - mdns: boolean; + mdns?: boolean; connectToDiscv5Bootnodes?: boolean; version?: string; private?: boolean; @@ -25,17 +24,14 @@ export interface NetworkOptions export const defaultNetworkOptions: NetworkOptions = { maxPeers: 55, // Allow some room above targetPeers for new inbound peers targetPeers: 50, - discv5FirstQueryDelayMs: 1000, localMultiaddrs: ["/ip4/0.0.0.0/tcp/9000"], bootMultiaddrs: [], - mdns: false, /** disabled by default */ discv5: null, rateLimitMultiplier: 1, // TODO: this value is 12 per spec, however lodestar has performance issue if there are too many mesh peers // see https://github.com/ChainSafe/lodestar/issues/5420 gossipsubDHigh: 9, - ...defaultGossipHandlerOpts, // TODO set to false in order to release 1.9.0 in a timely manner useWorker: false, }; diff --git a/packages/beacon-node/src/network/peers/peerManager.ts b/packages/beacon-node/src/network/peers/peerManager.ts index 1b5176cba1fb..a40610584919 100644 --- a/packages/beacon-node/src/network/peers/peerManager.ts +++ b/packages/beacon-node/src/network/peers/peerManager.ts @@ -43,6 +43,8 @@ const STATUS_INBOUND_GRACE_PERIOD = 15 * 1000; const CHECK_PING_STATUS_INTERVAL = 10 * 1000; /** A peer is considered long connection if it's >= 1 day */ const LONG_PEER_CONNECTION_MS = 24 * 60 * 60 * 1000; +/** Ref https://github.com/ChainSafe/lodestar/issues/3423 */ +const DEFAULT_DISCV5_FIRST_QUERY_DELAY_MS = 1000; /** * Tag peer when it's relevant and connecting to our node. * When node has > maxPeer (55), libp2p randomly prune peers if we don't tag peers in use. @@ -71,7 +73,7 @@ export type PeerManagerOpts = { * Delay the 1st query after starting discv5 * See https://github.com/ChainSafe/lodestar/issues/3423 */ - discv5FirstQueryDelayMs: number; + discv5FirstQueryDelayMs?: number; /** * If null, Don't run discv5 queries, nor connect to cached peers in the peerStore */ @@ -189,7 +191,7 @@ export class PeerManager { const discovery = opts.discv5 ? await PeerDiscovery.init(modules, { maxPeers: opts.maxPeers, - discv5FirstQueryDelayMs: opts.discv5FirstQueryDelayMs, + discv5FirstQueryDelayMs: opts.discv5FirstQueryDelayMs ?? DEFAULT_DISCV5_FIRST_QUERY_DELAY_MS, discv5: opts.discv5, connectToDiscv5Bootnodes: opts.connectToDiscv5Bootnodes, }) diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 72ae32cc8887..06050cfa0ff5 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -45,15 +45,8 @@ import {AggregatorTracker} from "./aggregatorTracker.js"; * Gossip handler options as part of network options */ export type GossipHandlerOpts = { - dontSendGossipAttestationsToForkchoice: boolean; -}; - -/** - * By default: - * + pass gossip attestations to forkchoice - */ -export const defaultGossipHandlerOpts = { - dontSendGossipAttestationsToForkchoice: false, + /** By default pass gossip attestations to forkchoice */ + dontSendGossipAttestationsToForkchoice?: boolean; }; export type ValidatorFnsModules = { diff --git a/packages/beacon-node/src/node/nodejs.ts b/packages/beacon-node/src/node/nodejs.ts index e5f737945595..49d4b805c594 100644 --- a/packages/beacon-node/src/node/nodejs.ts +++ b/packages/beacon-node/src/node/nodejs.ts @@ -187,10 +187,11 @@ export class BeaconNode { } const monitoring = opts.monitoring.endpoint - ? new MonitoringService("beacon", opts.monitoring, { - register: (metrics as Metrics).register, - logger: logger.child({module: LoggerModule.monitoring}), - }) + ? new MonitoringService( + "beacon", + {...opts.monitoring, endpoint: opts.monitoring.endpoint}, + {register: (metrics as Metrics).register, logger: logger.child({module: LoggerModule.monitoring})} + ) : null; const chain = new BeaconChain(opts.chain, { diff --git a/packages/beacon-node/src/node/options.ts b/packages/beacon-node/src/node/options.ts index 4ab872c4642a..475a4debee63 100644 --- a/packages/beacon-node/src/node/options.ts +++ b/packages/beacon-node/src/node/options.ts @@ -8,13 +8,18 @@ import {defaultNetworkOptions, NetworkOptions} from "../network/options.js"; import {defaultSyncOptions, SyncOptions} from "../sync/options.js"; import { defaultExecutionEngineOpts, + defaultExecutionEngineHttpOpts, ExecutionEngineOpts, ExecutionBuilderOpts, defaultExecutionBuilderOpts, + defaultExecutionBuilderHttpOpts, } from "../execution/index.js"; // Re-export so the CLI doesn't need to depend on lodestar-api export {allNamespaces} from "../api/rest/index.js"; +// Re-export to use as default values in CLI args +export {defaultExecutionEngineHttpOpts, defaultExecutionBuilderHttpOpts}; + export interface IBeaconNodeOptions { api: ApiOptions; chain: IChainOptions; diff --git a/packages/cli/src/cmds/beacon/options.ts b/packages/cli/src/cmds/beacon/options.ts index 0639b39e8981..27a7762e5b2d 100644 --- a/packages/cli/src/cmds/beacon/options.ts +++ b/packages/cli/src/cmds/beacon/options.ts @@ -20,6 +20,7 @@ type BeaconExtraArgs = { peerStoreDir?: string; persistNetworkIdentity?: boolean; private?: boolean; + attachToGlobalThis?: boolean; }; export const beaconExtraOptions: CliCommandOptions = { @@ -118,6 +119,12 @@ export const beaconExtraOptions: CliCommandOptions = { description: "Do not send implementation details over p2p identify protocol and in builder requests", type: "boolean", }, + + attachToGlobalThis: { + hidden: true, + description: "Attach the beacon node to `globalThis`. Useful to inspect a running beacon node.", + type: "boolean", + }, }; type ENRArgs = { @@ -168,16 +175,7 @@ const enrOptions: Record = { }, }; -export type DebugArgs = {attachToGlobalThis: boolean}; -export const debugOptions: CliCommandOptions = { - attachToGlobalThis: { - hidden: true, - description: "Attach the beacon node to `globalThis`. Useful to inspect a running beacon node.", - type: "boolean", - }, -}; - -export type BeaconArgs = BeaconExtraArgs & LogArgs & BeaconPaths & BeaconNodeArgs & ENRArgs & DebugArgs; +export type BeaconArgs = BeaconExtraArgs & LogArgs & BeaconPaths & BeaconNodeArgs & ENRArgs; export const beaconOptions: {[k: string]: Options} = { ...beaconExtraOptions, @@ -185,5 +183,4 @@ export const beaconOptions: {[k: string]: Options} = { ...beaconNodeOptions, ...paramsOptions, ...enrOptions, - ...debugOptions, }; diff --git a/packages/cli/src/cmds/lightclient/handler.ts b/packages/cli/src/cmds/lightclient/handler.ts index 11e5ed743d54..030aee032f48 100644 --- a/packages/cli/src/cmds/lightclient/handler.ts +++ b/packages/cli/src/cmds/lightclient/handler.ts @@ -7,6 +7,7 @@ import {getNodeLogger} from "@lodestar/logger/node"; import {getBeaconConfigFromArgs} from "../../config/beaconParams.js"; import {getGlobalPaths} from "../../paths/global.js"; import {parseLoggerArgs} from "../../util/logger.js"; +import {YargsError} from "../../util/errors.js"; import {GlobalArgs} from "../../options/index.js"; import {ILightClientArgs} from "./options.js"; @@ -17,7 +18,11 @@ export async function lightclientHandler(args: ILightClientArgs & GlobalArgs): P const logger = getNodeLogger( parseLoggerArgs(args, {defaultLogFilepath: path.join(globalPaths.dataDir, "lightclient.log")}, config) ); + const {beaconApiUrl, checkpointRoot} = args; + if (!beaconApiUrl) throw new YargsError("must provide beaconApiUrl arg"); + if (!checkpointRoot) throw new YargsError("must provide checkpointRoot arg"); + const api = getClient({baseUrl: beaconApiUrl}, {config}); const res = await api.beacon.getGenesis(); ApiError.assert(res, "Can not fetch genesis data"); diff --git a/packages/cli/src/cmds/lightclient/options.ts b/packages/cli/src/cmds/lightclient/options.ts index ba68a7aa5a5c..1dd1ddab8f00 100644 --- a/packages/cli/src/cmds/lightclient/options.ts +++ b/packages/cli/src/cmds/lightclient/options.ts @@ -2,8 +2,8 @@ import {LogArgs, logOptions} from "../../options/logOptions.js"; import {CliCommandOptions} from "../../util/index.js"; export type ILightClientArgs = LogArgs & { - beaconApiUrl: string; - checkpointRoot: string; + beaconApiUrl?: string; + checkpointRoot?: string; }; export const lightclientOptions: CliCommandOptions = { @@ -11,11 +11,9 @@ export const lightclientOptions: CliCommandOptions = { beaconApiUrl: { description: "Url to a beacon node that support lightclient API", type: "string", - require: true, }, checkpointRoot: { description: "Checkpoint root hex string to sync the lightclient from, start with 0x", type: "string", - require: true, }, }; diff --git a/packages/cli/src/cmds/validator/blsToExecutionChange.ts b/packages/cli/src/cmds/validator/blsToExecutionChange.ts index 59c184680cea..9f298d10d8a1 100644 --- a/packages/cli/src/cmds/validator/blsToExecutionChange.ts +++ b/packages/cli/src/cmds/validator/blsToExecutionChange.ts @@ -7,7 +7,7 @@ import {fromHexString} from "@chainsafe/ssz"; import bls from "@chainsafe/bls"; import {PointFormat} from "@chainsafe/bls/types"; -import {CliCommand} from "../../util/index.js"; +import {CliCommand, YargsError} from "../../util/index.js"; import {GlobalArgs} from "../../options/index.js"; import {getBeaconConfigFromArgs} from "../../config/index.js"; import {IValidatorCliArgs} from "./options.js"; @@ -15,9 +15,9 @@ import {IValidatorCliArgs} from "./options.js"; /* eslint-disable no-console */ type BlsToExecutionChangeArgs = { - publicKey: string; - fromBlsPrivkey: string; - toExecutionAddress: string; + publicKey?: string; + fromBlsPrivkey?: string; + toExecutionAddress?: string; }; export const blsToExecutionChange: CliCommand = { @@ -54,7 +54,11 @@ like to choose for BLS To Execution Change.", }, handler: async (args) => { - const publicKey = args.publicKey; + const {publicKey, fromBlsPrivkey, toExecutionAddress} = args; + if (!publicKey) throw new YargsError("must provide publicKey arg"); + if (!fromBlsPrivkey) throw new YargsError("must provide fromBlsPrivkey arg"); + if (!toExecutionAddress) throw new YargsError("must provide toExecutionAddress arg"); + // Fetch genesisValidatorsRoot always from beacon node as anyway beacon node is needed for // submitting the signed message const {config: chainForkConfig} = getBeaconConfigFromArgs(args); @@ -72,13 +76,13 @@ like to choose for BLS To Execution Change.", throw new Error(`Validator pubkey ${publicKey} not found in state`); } - const fromBlsPrivkey = bls.SecretKey.fromBytes(fromHexString(args.fromBlsPrivkey)); - const fromBlsPubkey = fromBlsPrivkey.toPublicKey().toBytes(PointFormat.compressed); + const blsPrivkey = bls.SecretKey.fromBytes(fromHexString(fromBlsPrivkey)); + const fromBlsPubkey = blsPrivkey.toPublicKey().toBytes(PointFormat.compressed); const blsToExecutionChange: capella.BLSToExecutionChange = { validatorIndex: stateValidator.index, fromBlsPubkey, - toExecutionAddress: fromHexString(args.toExecutionAddress), + toExecutionAddress: fromHexString(toExecutionAddress), }; const signatureFork = ForkName.phase0; @@ -86,7 +90,7 @@ like to choose for BLS To Execution Change.", const signingRoot = computeSigningRoot(ssz.capella.BLSToExecutionChange, blsToExecutionChange, domain); const signedBLSToExecutionChange = { message: blsToExecutionChange, - signature: fromBlsPrivkey.sign(signingRoot).toBytes(), + signature: blsPrivkey.sign(signingRoot).toBytes(), }; ApiError.assert( diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index f92814cdcb3d..2b6fb28ab16d 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -32,8 +32,8 @@ export type IValidatorCliArgs = AccountValidatorArgs & LogArgs & { validatorsDbDir?: string; beaconNodes: string[]; - force: boolean; - graffiti: string; + force?: boolean; + graffiti?: string; afterBlockDelaySlotFraction?: number; scAfterBlockDelaySlotFraction?: number; disableAttestationGrouping?: boolean; diff --git a/packages/cli/src/cmds/validator/slashingProtection/export.ts b/packages/cli/src/cmds/validator/slashingProtection/export.ts index 97d3768d5ad6..e6dc3af49016 100644 --- a/packages/cli/src/cmds/validator/slashingProtection/export.ts +++ b/packages/cli/src/cmds/validator/slashingProtection/export.ts @@ -13,7 +13,7 @@ import {getGenesisValidatorsRoot, getSlashingProtection} from "./utils.js"; import {ISlashingProtectionArgs} from "./options.js"; type ExportArgs = { - file: string; + file?: string; pubkeys?: string[]; }; @@ -50,6 +50,9 @@ export const exportCmd: CliCommand { + const {file} = args; + if (!file) throw new YargsError("must provide file arg"); + const {config, network} = getBeaconConfigFromArgs(args); const validatorPaths = getValidatorPaths(args, network); // slashingProtection commands are fast so do not require logFile feature @@ -102,8 +105,8 @@ export const exportCmd: CliCommand = @@ -38,6 +38,9 @@ export const importCmd: CliCommand { + const {file} = args; + if (!file) throw new YargsError("must provide file arg"); + const {config, network} = getBeaconConfigFromArgs(args); const validatorPaths = getValidatorPaths(args, network); // slashingProtection commands are fast so do not require logFile feature @@ -58,8 +61,8 @@ export const importCmd: CliCommand = { rest: { type: "boolean", description: "Enable/disable HTTP API", - defaultDescription: String(defaultOptions.api.rest.enabled), + default: defaultOptions.api.rest.enabled, group: "api", }, @@ -76,7 +76,7 @@ export const options: CliCommandOptions = { "rest.port": { type: "number", description: "Set port for HTTP API", - defaultDescription: String(defaultOptions.api.rest.port), + default: defaultOptions.api.rest.port, group: "api", }, "rest.headerLimit": { diff --git a/packages/cli/src/options/beaconNodeOptions/builder.ts b/packages/cli/src/options/beaconNodeOptions/builder.ts index 533dc6124dd9..7ef671939dcb 100644 --- a/packages/cli/src/options/beaconNodeOptions/builder.ts +++ b/packages/cli/src/options/beaconNodeOptions/builder.ts @@ -1,18 +1,18 @@ -import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; +import {defaultExecutionBuilderHttpOpts, IBeaconNodeOptions} from "@lodestar/beacon-node"; import {CliCommandOptions} from "../../util/index.js"; export type ExecutionBuilderArgs = { builder: boolean; - "builder.urls": string[]; - "builder.timeout": number; - "builder.faultInspectionWindow": number; - "builder.allowedFaults": number; + "builder.urls"?: string[]; + "builder.timeout"?: number; + "builder.faultInspectionWindow"?: number; + "builder.allowedFaults"?: number; }; export function parseArgs(args: ExecutionBuilderArgs): IBeaconNodeOptions["executionBuilder"] { return { enabled: args["builder"], - urls: args["builder.urls"], + urls: args["builder.urls"] ?? defaultExecutionBuilderHttpOpts.urls, timeout: args["builder.timeout"], faultInspectionWindow: args["builder.faultInspectionWindow"], allowedFaults: args["builder.allowedFaults"], @@ -23,16 +23,13 @@ export const options: CliCommandOptions = { builder: { description: "Enable builder interface", type: "boolean", - defaultDescription: `${ - defaultOptions.executionBuilder.mode === "http" ? defaultOptions.executionBuilder.enabled : false - }`, + default: defaultExecutionBuilderHttpOpts.enabled, group: "builder", }, "builder.urls": { description: "Urls hosting the builder API", - defaultDescription: - defaultOptions.executionBuilder.mode === "http" ? defaultOptions.executionBuilder.urls.join(" ") : "", + defaultDescription: defaultExecutionBuilderHttpOpts.urls.join(","), type: "array", string: true, coerce: (urls: string[]): string[] => @@ -44,8 +41,7 @@ export const options: CliCommandOptions = { "builder.timeout": { description: "Timeout in milliseconds for builder API HTTP client", type: "number", - defaultDescription: - defaultOptions.executionBuilder.mode === "http" ? String(defaultOptions.executionBuilder.timeout) : "", + defaultDescription: String(defaultExecutionBuilderHttpOpts.timeout), group: "builder", }, diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index 22f3896bb318..55e9b8697f1d 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -4,24 +4,24 @@ import {CliCommandOptions} from "../../util/index.js"; export type ChainArgs = { suggestedFeeRecipient: string; - "chain.blsVerifyAllMultiThread": boolean; - "chain.blsVerifyAllMainThread": boolean; - "chain.disableBlsBatchVerify": boolean; - "chain.persistInvalidSszObjects": boolean; + "chain.blsVerifyAllMultiThread"?: boolean; + "chain.blsVerifyAllMainThread"?: boolean; + "chain.disableBlsBatchVerify"?: boolean; + "chain.persistInvalidSszObjects"?: boolean; // No need to define chain.persistInvalidSszObjects as part of ChainArgs // as this is defined as part of BeaconPaths // "chain.persistInvalidSszObjectsDir": string; - "chain.proposerBoostEnabled": boolean; - "chain.disableImportExecutionFcU": boolean; - "chain.preaggregateSlotDistance": number; - "chain.attDataCacheSlotDistance": number; - "chain.computeUnrealized": boolean; - "chain.assertCorrectProgressiveBalances": boolean; - "chain.maxSkipSlots": number; - "chain.trustedSetup": string; + "chain.proposerBoostEnabled"?: boolean; + "chain.disableImportExecutionFcU"?: boolean; + "chain.preaggregateSlotDistance"?: number; + "chain.attDataCacheSlotDistance"?: number; + "chain.computeUnrealized"?: boolean; + "chain.assertCorrectProgressiveBalances"?: boolean; + "chain.maxSkipSlots"?: number; + "chain.trustedSetup"?: string; "safe-slots-to-import-optimistically": number; "chain.archiveStateEpochFrequency": number; - emitPayloadAttributes: boolean; + emitPayloadAttributes?: boolean; }; export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] { @@ -52,7 +52,7 @@ export const options: CliCommandOptions = { type: "string", description: "Specify fee recipient default for collecting the EL block fees and rewards (a hex string representing 20 bytes address: ^0x[a-fA-F0-9]{40}$) in case validator fails to update for a validator index before calling produceBlock.", - defaultDescription: defaultOptions.chain.suggestedFeeRecipient, + default: defaultOptions.chain.suggestedFeeRecipient, group: "chain", }, @@ -160,13 +160,14 @@ Will double processing times. Use only for debugging purposes.", type: "number", description: "Slots from current (clock) slot till which its safe to import a block optimistically if the merge is not justified yet.", - defaultDescription: String(defaultOptions.chain.safeSlotsToImportOptimistically), + default: defaultOptions.chain.safeSlotsToImportOptimistically, group: "chain", }, "chain.archiveStateEpochFrequency": { hidden: true, description: "Minimum number of epochs between archived states", + default: defaultOptions.chain.archiveStateEpochFrequency, type: "number", group: "chain", }, diff --git a/packages/cli/src/options/beaconNodeOptions/eth1.ts b/packages/cli/src/options/beaconNodeOptions/eth1.ts index 932f372caeb2..aa47c6c3b1a7 100644 --- a/packages/cli/src/options/beaconNodeOptions/eth1.ts +++ b/packages/cli/src/options/beaconNodeOptions/eth1.ts @@ -4,12 +4,12 @@ import {CliCommandOptions, extractJwtHexSecret} from "../../util/index.js"; import {ExecutionEngineArgs} from "./execution.js"; export type Eth1Args = { - eth1: boolean; - "eth1.providerUrls": string[]; - "eth1.depositContractDeployBlock": number; - "eth1.disableEth1DepositDataTracker": boolean; - "eth1.unsafeAllowDepositDataOverwrite": boolean; - "eth1.forcedEth1DataVote": string; + eth1?: boolean; + "eth1.providerUrls"?: string[]; + "eth1.depositContractDeployBlock"?: number; + "eth1.disableEth1DepositDataTracker"?: boolean; + "eth1.unsafeAllowDepositDataOverwrite"?: boolean; + "eth1.forcedEth1DataVote"?: string; }; export function parseArgs(args: Eth1Args & Partial): IBeaconNodeOptions["eth1"] { @@ -49,7 +49,7 @@ export const options: CliCommandOptions = { "eth1.providerUrls": { description: "Urls to Eth1 node with enabled rpc. If not explicity provided and execution endpoint provided via execution.urls, it will use execution.urls. Otherwise will try connecting on the specified default(s)", - defaultDescription: defaultOptions.eth1.providerUrls.join(" "), + defaultDescription: defaultOptions.eth1.providerUrls?.join(","), type: "array", string: true, coerce: (urls: string[]): string[] => diff --git a/packages/cli/src/options/beaconNodeOptions/execution.ts b/packages/cli/src/options/beaconNodeOptions/execution.ts index 39f48e139739..b1faf482ade8 100644 --- a/packages/cli/src/options/beaconNodeOptions/execution.ts +++ b/packages/cli/src/options/beaconNodeOptions/execution.ts @@ -1,10 +1,10 @@ import fs from "node:fs"; -import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; +import {defaultExecutionEngineHttpOpts, IBeaconNodeOptions} from "@lodestar/beacon-node"; import {CliCommandOptions, extractJwtHexSecret} from "../../util/index.js"; export type ExecutionEngineArgs = { "execution.urls": string[]; - "execution.timeout": number; + "execution.timeout"?: number; "execution.retryAttempts": number; "execution.retryDelay": number; "execution.engineMock"?: boolean; @@ -37,8 +37,7 @@ export function parseArgs(args: ExecutionEngineArgs): IBeaconNodeOptions["execut export const options: CliCommandOptions = { "execution.urls": { description: "Urls to execution client engine API", - defaultDescription: - defaultOptions.executionEngine.mode === "http" ? defaultOptions.executionEngine.urls.join(" ") : "", + default: defaultExecutionEngineHttpOpts.urls.join(","), type: "array", string: true, coerce: (urls: string[]): string[] => @@ -50,24 +49,21 @@ export const options: CliCommandOptions = { "execution.timeout": { description: "Timeout in milliseconds for execution engine API HTTP client", type: "number", - defaultDescription: - defaultOptions.executionEngine.mode === "http" ? String(defaultOptions.executionEngine.timeout) : "", + defaultDescription: String(defaultExecutionEngineHttpOpts.timeout), group: "execution", }, "execution.retryAttempts": { description: "Number of retry attempts when calling execution engine API", type: "number", - defaultDescription: - defaultOptions.executionEngine.mode === "http" ? String(defaultOptions.executionEngine.retryAttempts) : "1", + default: defaultExecutionEngineHttpOpts.retryAttempts, group: "execution", }, "execution.retryDelay": { description: "Delay time in milliseconds between retries when retrying calls to the execution engine API", type: "number", - defaultDescription: - defaultOptions.executionEngine.mode === "http" ? String(defaultOptions.executionEngine.retryDelay) : "0", + default: defaultExecutionEngineHttpOpts.retryDelay, group: "execution", }, diff --git a/packages/cli/src/options/beaconNodeOptions/metrics.ts b/packages/cli/src/options/beaconNodeOptions/metrics.ts index c10e3d1dd844..dc328cfa5685 100644 --- a/packages/cli/src/options/beaconNodeOptions/metrics.ts +++ b/packages/cli/src/options/beaconNodeOptions/metrics.ts @@ -4,7 +4,7 @@ import {CliCommandOptions} from "../../util/index.js"; export type MetricsArgs = { metrics: boolean; "metrics.port": number; - "metrics.address": string; + "metrics.address"?: string; }; export function parseArgs(args: MetricsArgs): IBeaconNodeOptions["metrics"] { @@ -19,14 +19,14 @@ export const options: CliCommandOptions = { metrics: { type: "boolean", description: "Enable the Prometheus metrics HTTP server", - defaultDescription: String(defaultOptions.metrics.enabled), + default: defaultOptions.metrics.enabled, group: "metrics", }, "metrics.port": { type: "number", description: "Listen TCP port for the Prometheus metrics HTTP server", - defaultDescription: String(defaultOptions.metrics.port), + default: defaultOptions.metrics.port, group: "metrics", }, diff --git a/packages/cli/src/options/beaconNodeOptions/monitoring.ts b/packages/cli/src/options/beaconNodeOptions/monitoring.ts index 7517bfb312dc..f9224dca684f 100644 --- a/packages/cli/src/options/beaconNodeOptions/monitoring.ts +++ b/packages/cli/src/options/beaconNodeOptions/monitoring.ts @@ -2,11 +2,11 @@ import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; import {CliCommandOptions} from "../../util/index.js"; export type MonitoringArgs = { - "monitoring.endpoint": string; - "monitoring.interval": number; - "monitoring.initialDelay": number; - "monitoring.requestTimeout": number; - "monitoring.collectSystemStats": boolean; + "monitoring.endpoint"?: string; + "monitoring.interval"?: number; + "monitoring.initialDelay"?: number; + "monitoring.requestTimeout"?: number; + "monitoring.collectSystemStats"?: boolean; }; export function parseArgs(args: MonitoringArgs): IBeaconNodeOptions["monitoring"] { diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 9580b02e79e2..b822e270fe7c 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -10,31 +10,31 @@ export type NetworkArgs = { port?: number; discoveryPort?: number; bootnodes?: string[]; - targetPeers: number; - subscribeAllSubnets: boolean; - disablePeerScoring: boolean; - mdns: boolean; - "network.maxPeers": number; - "network.connectToDiscv5Bootnodes": boolean; - "network.discv5FirstQueryDelayMs": number; - "network.dontSendGossipAttestationsToForkchoice": boolean; - "network.allowPublishToZeroPeers": boolean; - "network.gossipsubD": number; - "network.gossipsubDLow": number; - "network.gossipsubDHigh": number; - "network.gossipsubAwaitHandler": boolean; - "network.rateLimitMultiplier": number; + targetPeers?: number; + subscribeAllSubnets?: boolean; + disablePeerScoring?: boolean; + mdns?: boolean; + "network.maxPeers"?: number; + "network.connectToDiscv5Bootnodes"?: boolean; + "network.discv5FirstQueryDelayMs"?: number; + "network.dontSendGossipAttestationsToForkchoice"?: boolean; + "network.allowPublishToZeroPeers"?: boolean; + "network.gossipsubD"?: number; + "network.gossipsubDLow"?: number; + "network.gossipsubDHigh"?: number; + "network.gossipsubAwaitHandler"?: boolean; + "network.rateLimitMultiplier"?: number; "network.maxGossipTopicConcurrency"?: number; - "network.useWorker": boolean; + "network.useWorker"?: boolean; /** @deprecated This option is deprecated and should be removed in next major release. */ - "network.requestCountPeerLimit": number; + "network.requestCountPeerLimit"?: number; /** @deprecated This option is deprecated and should be removed in next major release. */ - "network.blockCountTotalLimit": number; + "network.blockCountTotalLimit"?: number; /** @deprecated This option is deprecated and should be removed in next major release. */ - "network.blockCountPeerLimit": number; + "network.blockCountPeerLimit"?: number; /** @deprecated This option is deprecated and should be removed in next major release. */ - "network.rateTrackerTimeoutMs": number; + "network.rateTrackerTimeoutMs"?: number; }; export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { @@ -60,8 +60,8 @@ export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { enr: undefined as any, } : null, - maxPeers, - targetPeers, + maxPeers: maxPeers ?? defaultOptions.network.maxPeers, + targetPeers: targetPeers ?? defaultOptions.network.targetPeers, localMultiaddrs: [`/ip4/${listenAddress}/tcp/${tcpPort}`], subscribeAllSubnets: args["subscribeAllSubnets"], disablePeerScoring: args["disablePeerScoring"], diff --git a/packages/cli/src/options/beaconNodeOptions/sync.ts b/packages/cli/src/options/beaconNodeOptions/sync.ts index 0b87ad22ac39..6a28338adad3 100644 --- a/packages/cli/src/options/beaconNodeOptions/sync.ts +++ b/packages/cli/src/options/beaconNodeOptions/sync.ts @@ -2,17 +2,17 @@ import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; import {CliCommandOptions} from "../../util/index.js"; export type SyncArgs = { - "sync.isSingleNode": boolean; - "sync.disableProcessAsChainSegment": boolean; - "sync.disableRangeSync": boolean; - "sync.backfillBatchSize": number; + "sync.isSingleNode"?: boolean; + "sync.disableProcessAsChainSegment"?: boolean; + "sync.disableRangeSync"?: boolean; + "sync.backfillBatchSize"?: number; }; export function parseArgs(args: SyncArgs): IBeaconNodeOptions["sync"] { return { isSingleNode: args["sync.isSingleNode"], disableProcessAsChainSegment: args["sync.disableProcessAsChainSegment"], - backfillBatchSize: args["sync.backfillBatchSize"], + backfillBatchSize: args["sync.backfillBatchSize"] ?? defaultOptions.sync.backfillBatchSize, disableRangeSync: args["sync.disableRangeSync"], }; } diff --git a/packages/cli/src/options/globalOptions.ts b/packages/cli/src/options/globalOptions.ts index a70799c30de4..30e515de49c0 100644 --- a/packages/cli/src/options/globalOptions.ts +++ b/packages/cli/src/options/globalOptions.ts @@ -6,7 +6,7 @@ import {paramsOptions, IParamsArgs} from "./paramsOptions.js"; type GlobalSingleArgs = { dataDir?: string; network?: NetworkName; - paramsFile: string; + paramsFile?: string; preset: string; presetFile?: string; }; diff --git a/packages/cli/src/util/command.ts b/packages/cli/src/util/command.ts index f22aca319af0..32d7b24e02bf 100644 --- a/packages/cli/src/util/command.ts +++ b/packages/cli/src/util/command.ts @@ -1,6 +1,11 @@ import {Options, Argv} from "yargs"; -export type CliCommandOptions = Required<{[key in keyof OwnArgs]: Options}>; +export type CliCommandOptions = Required<{ + [K in keyof OwnArgs]: undefined extends OwnArgs[K] + ? Options + : // If arg cannot be undefined it must specify a default value + Options & Required>; +}>; // eslint-disable-next-line @typescript-eslint/no-explicit-any export interface CliCommand, ParentArgs = Record, R = any> { From 4bbe595f5ddaf9abfc1bf65d7a961fca0b6b2f35 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 20 Jun 2023 14:49:19 +0200 Subject: [PATCH 10/96] chore: fix lint-grafana-dashboard.mjs id assertion (#5673) --- scripts/lint-grafana-dashboard.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lint-grafana-dashboard.mjs b/scripts/lint-grafana-dashboard.mjs index 70b93fe0df0c..9cee9b6a5694 100644 --- a/scripts/lint-grafana-dashboard.mjs +++ b/scripts/lint-grafana-dashboard.mjs @@ -132,7 +132,7 @@ export function lintGrafanaDashboard(json) { json.graphTooltip = 1; // null id to match "Export for sharing externally" format, only set to null if set to respect order - if (json !== undefined) { + if (json.id !== undefined) { json.id = null; } From f6774341d520a0bc3e735417a275313286a59c9b Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 20 Jun 2023 16:04:16 +0200 Subject: [PATCH 11/96] fix!: update validator liveness API to be spec compliant (#5667) --- packages/api/src/beacon/routes/validator.ts | 20 +++++---- .../test/unit/beacon/testData/validator.ts | 2 +- .../src/api/impl/validator/index.ts | 10 ++--- .../test/e2e/api/lodestar/lodestar.test.ts | 22 +++++----- .../src/services/doppelgangerService.ts | 41 ++++++++----------- .../test/unit/services/doppleganger.test.ts | 6 +-- 6 files changed, 49 insertions(+), 52 deletions(-) diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index 9b96263afee6..1c399d71ff72 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -132,7 +132,6 @@ export type SyncCommitteeSelection = { export type LivenessResponseData = { index: ValidatorIndex; - epoch: Epoch; isLive: boolean; }; @@ -390,9 +389,9 @@ export type Api = { /** Returns validator indices that have been observed to be active on the network */ getLiveness( - indices: ValidatorIndex[], - epoch: Epoch - ): Promise>; + epoch: Epoch, + validatorIndices: ValidatorIndex[] + ): Promise>; registerValidator( registrations: bellatrix.SignedValidatorRegistrationV1[] @@ -419,7 +418,7 @@ export const routesData: RoutesData = { prepareBeaconProposer: {url: "/eth/v1/validator/prepare_beacon_proposer", method: "POST"}, submitBeaconCommitteeSelections: {url: "/eth/v1/validator/beacon_committee_selections", method: "POST"}, submitSyncCommitteeSelections: {url: "/eth/v1/validator/sync_committee_selections", method: "POST"}, - getLiveness: {url: "/eth/v1/validator/liveness", method: "GET"}, + getLiveness: {url: "/eth/v1/validator/liveness/{epoch}", method: "POST"}, registerValidator: {url: "/eth/v1/validator/register_validator", method: "POST"}, }; @@ -441,7 +440,7 @@ export type ReqTypes = { prepareBeaconProposer: {body: unknown}; submitBeaconCommitteeSelections: {body: unknown}; submitSyncCommitteeSelections: {body: unknown}; - getLiveness: {query: {indices: ValidatorIndex[]; epoch: Epoch}}; + getLiveness: {params: {epoch: Epoch}; body: U64Str[]}; registerValidator: {body: unknown}; }; @@ -578,9 +577,12 @@ export function getReqSerializers(): ReqSerializers { parseReq: () => [[]], }, getLiveness: { - writeReq: (indices, epoch) => ({query: {indices, epoch}}), - parseReq: ({query}) => [query.indices, query.epoch], - schema: {query: {indices: Schema.UintArray, epoch: Schema.Uint}}, + writeReq: (epoch, indexes) => ({params: {epoch}, body: indexes.map((i) => toU64Str(i))}), + parseReq: ({params, body}) => [params.epoch, body.map((i) => fromU64Str(i))], + schema: { + params: {epoch: Schema.UintRequired}, + body: Schema.StringArray, + }, }, registerValidator: reqOnlyBody(ArrayOf(ssz.bellatrix.SignedValidatorRegistrationV1), Schema.ObjectArray), }; diff --git a/packages/api/test/unit/beacon/testData/validator.ts b/packages/api/test/unit/beacon/testData/validator.ts index 90792f467d7f..9cc92f5a8629 100644 --- a/packages/api/test/unit/beacon/testData/validator.ts +++ b/packages/api/test/unit/beacon/testData/validator.ts @@ -100,7 +100,7 @@ export const testData: GenericServerTestCases = { res: {data: [{validatorIndex: 1, slot: 2, subcommitteeIndex: 3, selectionProof}]}, }, getLiveness: { - args: [[0], 0], + args: [0, [0]], res: {data: []}, }, registerValidator: { diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 14909a0b0c64..def416fc963a 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -731,23 +731,23 @@ export function getValidatorApi({ throw new OnlySupportedByDVT(); }, - async getLiveness(indices: ValidatorIndex[], epoch: Epoch) { - if (indices.length === 0) { + async getLiveness(epoch, validatorIndices) { + if (validatorIndices.length === 0) { return { data: [], }; } const currentEpoch = chain.clock.currentEpoch; if (epoch < currentEpoch - 1 || epoch > currentEpoch + 1) { - throw new Error( + throw new ApiError( + 400, `Request epoch ${epoch} is more than one epoch before or after the current epoch ${currentEpoch}` ); } return { - data: indices.map((index: ValidatorIndex) => ({ + data: validatorIndices.map((index) => ({ index, - epoch, isLive: chain.validatorSeenAtEpoch(index, epoch), })), }; diff --git a/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts b/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts index 11f2566a29e7..2a79daae914c 100644 --- a/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts +++ b/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts @@ -65,15 +65,15 @@ describe("api / impl / validator", function () { const client = getClient({baseUrl: `http://127.0.0.1:${restPort}`}, {config}); - await expect(client.validator.getLiveness([1, 2, 3, 4, 5], 0)).to.eventually.deep.equal( + await expect(client.validator.getLiveness(0, [1, 2, 3, 4, 5])).to.eventually.deep.equal( { response: { data: [ - {index: 1, epoch: 0, isLive: true}, - {index: 2, epoch: 0, isLive: true}, - {index: 3, epoch: 0, isLive: true}, - {index: 4, epoch: 0, isLive: true}, - {index: 5, epoch: 0, isLive: false}, + {index: 1, isLive: true}, + {index: 2, isLive: true}, + {index: 3, isLive: true}, + {index: 4, isLive: true}, + {index: 5, isLive: false}, ], }, ok: true, @@ -117,19 +117,19 @@ describe("api / impl / validator", function () { const previousEpoch = currentEpoch - 1; // current epoch is fine - await expect(client.validator.getLiveness([1], currentEpoch)).to.not.be.rejected; + await expect(client.validator.getLiveness(currentEpoch, [1])).to.not.be.rejected; // next epoch is fine - await expect(client.validator.getLiveness([1], nextEpoch)).to.not.be.rejected; + await expect(client.validator.getLiveness(nextEpoch, [1])).to.not.be.rejected; // previous epoch is fine - await expect(client.validator.getLiveness([1], previousEpoch)).to.not.be.rejected; + await expect(client.validator.getLiveness(previousEpoch, [1])).to.not.be.rejected; // more than next epoch is not fine - const res1 = await client.validator.getLiveness([1], currentEpoch + 2); + const res1 = await client.validator.getLiveness(currentEpoch + 2, [1]); expect(res1.ok).to.be.false; expect(res1.error?.message).to.include( `Request epoch ${currentEpoch + 2} is more than one epoch before or after the current epoch ${currentEpoch}` ); // more than previous epoch is not fine - const res2 = await client.validator.getLiveness([1], currentEpoch - 2); + const res2 = await client.validator.getLiveness(currentEpoch - 2, [1]); expect(res2.ok).to.be.false; expect(res2.error?.message).to.include( `Request epoch ${currentEpoch - 2} is more than one epoch before or after the current epoch ${currentEpoch}` diff --git a/packages/validator/src/services/doppelgangerService.ts b/packages/validator/src/services/doppelgangerService.ts index 7f656bf9aa2c..cb99200637c8 100644 --- a/packages/validator/src/services/doppelgangerService.ts +++ b/packages/validator/src/services/doppelgangerService.ts @@ -1,5 +1,5 @@ import {Epoch, ValidatorIndex} from "@lodestar/types"; -import {Api, ApiError} from "@lodestar/api"; +import {Api, ApiError, routes} from "@lodestar/api"; import {Logger, sleep} from "@lodestar/utils"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ProcessShutdownCallback, PubkeyHex} from "../types.js"; @@ -12,10 +12,10 @@ import {IndicesService} from "./indices.js"; const DEFAULT_REMAINING_DETECTION_EPOCHS = 1; const REMAINING_EPOCHS_IF_DOPPLEGANGER = Infinity; -export type LivenessResponseData = { - index: ValidatorIndex; +/** Liveness responses for a given epoch */ +type EpochLivenessData = { epoch: Epoch; - isLive: boolean; + responses: routes.validator.LivenessResponseData[]; }; export type DoppelgangerState = { @@ -127,34 +127,34 @@ export class DoppelgangerService { // in the remaining 25% of the last slot of the previous epoch const indicesToCheck = Array.from(indicesToCheckMap.keys()); const [previous, current] = await Promise.all([ - this.getLiveness(indicesToCheck, currentEpoch - 1), - this.getLiveness(indicesToCheck, currentEpoch), + this.getLiveness(currentEpoch - 1, indicesToCheck), + this.getLiveness(currentEpoch, indicesToCheck), ]); this.detectDoppelganger(currentEpoch, previous, current, indicesToCheckMap); }; - private async getLiveness(indicesToCheck: ValidatorIndex[], epoch: Epoch): Promise { + private async getLiveness(epoch: Epoch, indicesToCheck: ValidatorIndex[]): Promise { if (epoch < 0) { - return []; + return {epoch, responses: []}; } - const res = await this.api.validator.getLiveness(indicesToCheck, epoch); + const res = await this.api.validator.getLiveness(epoch, indicesToCheck); if (!res.ok) { this.logger.error( `Error getting liveness data for epoch ${epoch}`, {}, new ApiError(res.error.message ?? "", res.error.code, "validator.getLiveness") ); - return []; + return {epoch, responses: []}; } - return res.response.data; + return {epoch, responses: res.response.data}; } private detectDoppelganger( currentEpoch: Epoch, - previousEpochLiveness: LivenessResponseData[], - currentEpochLiveness: LivenessResponseData[], + previousEpochLiveness: EpochLivenessData, + currentEpochLiveness: EpochLivenessData, indicesToCheckMap: Map ): void { const previousEpoch = currentEpoch - 1; @@ -165,7 +165,7 @@ export class DoppelgangerService { // A following loop will update the states of each validator, depending on whether or not // any violators were detected here. - for (const responses of [previousEpochLiveness, currentEpochLiveness]) { + for (const {epoch, responses} of [previousEpochLiveness, currentEpochLiveness]) { for (const response of responses) { if (!response.isLive) { continue; @@ -177,7 +177,7 @@ export class DoppelgangerService { continue; } - if (state.nextEpochToCheck <= response.epoch) { + if (state.nextEpochToCheck <= epoch) { // Doppleganger detected violators.push(response.index); } @@ -209,21 +209,16 @@ export class DoppelgangerService { // // Do not bother iterating through the current epoch responses since they've already been // checked for violators and they don't result in updating the state. - for (const response of previousEpochLiveness) { - if (response.epoch !== previousEpoch) { - // Server sending bad data - throw Error(`Inconsistent livenessResponseData epoch ${response.epoch} != ${previousEpoch}`); - } - + for (const response of previousEpochLiveness.responses) { const state = this.doppelgangerStateByPubkey.get(indicesToCheckMap.get(response.index) ?? ""); if (!state) { this.logger.error(`Inconsistent livenessResponseData unknown index ${response.index}`); continue; } - if (!response.isLive && state.nextEpochToCheck <= response.epoch) { + if (!response.isLive && state.nextEpochToCheck <= previousEpoch) { state.remainingEpochs--; - state.nextEpochToCheck = response.epoch + 1; + state.nextEpochToCheck = currentEpoch; this.metrics?.doppelganger.epochsChecked.inc(1); const {remainingEpochs} = state; diff --git a/packages/validator/test/unit/services/doppleganger.test.ts b/packages/validator/test/unit/services/doppleganger.test.ts index 98e7e617c1df..af8abd79204e 100644 --- a/packages/validator/test/unit/services/doppleganger.test.ts +++ b/packages/validator/test/unit/services/doppleganger.test.ts @@ -145,15 +145,15 @@ type LivenessMap = Map>; function getMockBeaconApi(livenessMap: LivenessMap): Api { return { validator: { - async getLiveness(indices, epoch) { + async getLiveness(epoch, validatorIndices) { return { response: { - data: indices.map((index) => { + data: validatorIndices.map((index) => { const livenessEpoch = livenessMap.get(epoch); if (!livenessEpoch) throw Error(`Unknown epoch ${epoch}`); const isLive = livenessEpoch.get(index); if (isLive === undefined) throw Error(`No liveness for epoch ${epoch} index ${index}`); - return {index, epoch, isLive}; + return {index, isLive}; }), }, ok: true, From 5d9d86d10087060179201aba8bdc5fe42b02ddc5 Mon Sep 17 00:00:00 2001 From: g11tech Date: Tue, 20 Jun 2023 19:36:27 +0530 Subject: [PATCH 12/96] feat: add EL sim utility fn to send big raw blob txs (#5661) --- packages/beacon-node/test/utils/runEl.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/test/utils/runEl.ts b/packages/beacon-node/test/utils/runEl.ts index 48aebc305eda..9adf1a83c151 100644 --- a/packages/beacon-node/test/utils/runEl.ts +++ b/packages/beacon-node/test/utils/runEl.ts @@ -9,6 +9,7 @@ import {shell} from "../sim/shell.js"; /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable no-console */ +let txRpcId = 1; export enum ELStartMode { PreMerge = "pre-merge", @@ -219,10 +220,25 @@ export async function sendTransaction(url: string, transaction: Record { + fs.writeFileSync( + dataFilePath, + `{"jsonrpc":"2.0","method":"eth_sendRawTransaction","params":["${transactionRawHex}"],"id":${txRpcId++}}` + ); + await shell(`curl -d @${dataFilePath} -H "Content-Type: application/json" -X POST ${url}; rm ${dataFilePath}`); +} + export async function getBalance(url: string, account: string): Promise { const response: string = await shell( `curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_getBalance","params":["${account}","latest"],"id":67}' ${url}` From 61c838b84f1d4cd5deba20942d84609432fe8707 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 20 Jun 2023 11:48:13 -0400 Subject: [PATCH 13/96] fix: simplify snappy frame encoding (#5617) * fix: simplify snappy frame encoding * fix e2e tests --- packages/beacon-node/package.json | 2 - .../test/e2e/network/reqrespEncode.test.ts | 3 +- packages/reqresp/package.json | 4 +- .../encodingStrategies/sszSnappy/encode.ts | 21 +- .../sszSnappy/snappyFrames/common.ts | 9 + .../sszSnappy/snappyFrames/compress.ts | 54 ++++ .../sszSnappy/snappyFrames/uncompress.ts | 10 +- .../sszSnappy/snappyFrames/uncompress.test.ts | 47 +-- packages/reqresp/test/utils/index.ts | 5 +- types/snappy-stream/index.d.ts | 6 - types/stream-to-it/index.d.ts | 4 - yarn.lock | 269 ++++++------------ 12 files changed, 192 insertions(+), 242 deletions(-) create mode 100644 packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/common.ts create mode 100644 packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/compress.ts delete mode 100644 types/snappy-stream/index.d.ts delete mode 100644 types/stream-to-it/index.d.ts diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 4bc11784af7e..382400fc35ab 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -101,7 +101,6 @@ "@chainsafe/libp2p-gossipsub": "^6.2.0", "@chainsafe/libp2p-noise": "^11.0.4", "@chainsafe/persistent-merkle-tree": "^0.5.0", - "@chainsafe/snappy-stream": "^5.1.2", "@chainsafe/ssz": "^0.10.2", "@chainsafe/threads": "^1.11.0", "@ethersproject/abi": "^5.7.0", @@ -149,7 +148,6 @@ "prometheus-gc-stats": "^0.6.4", "qs": "^6.11.1", "snappyjs": "^0.7.0", - "stream-to-it": "^0.2.4", "strict-event-emitter-types": "^2.0.0", "systeminformation": "^5.17.12", "uint8arraylist": "^2.4.3", diff --git a/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts b/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts index e8c06c8f444f..4a550d938f8d 100644 --- a/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts +++ b/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts @@ -96,8 +96,9 @@ describe("reqresp encoder", () => { } const chunks = await all(stream.source); + const join = (c: string[]): string => c.join("").replace(/0x/g, ""); const chunksHex = chunks.map((chunk) => toHex(chunk.slice(0, chunk.byteLength))); - expect(chunksHex).deep.equals(expectedChunks, `not expected response to ${protocol}`); + expect(join(chunksHex)).deep.equals(join(expectedChunks), `not expected response to ${protocol}`); } it("assert correct handler switch between metadata v2 and v1", async () => { diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 627b1d15ae3f..b9e050cb9a9a 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -54,15 +54,15 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/snappy-stream": "^5.1.2", + "@chainsafe/fast-crc32c": "^4.1.1", "@libp2p/interface-connection": "^3.0.2", "@libp2p/interface-peer-id": "^2.0.1", "@lodestar/config": "^1.8.0", "@lodestar/params": "^1.8.0", "@lodestar/utils": "^1.8.0", "it-all": "^3.0.1", + "snappy": "^7.2.2", "snappyjs": "^0.7.0", - "stream-to-it": "^0.2.4", "uint8arraylist": "^2.4.3", "varint": "^6.0.0" }, diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts index 32a551f4aaa6..f3864cc25077 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts @@ -1,6 +1,5 @@ import varint from "varint"; -import {source} from "stream-to-it"; -import snappy from "@chainsafe/snappy-stream"; +import {encodeSnappy} from "./snappyFrames/compress.js"; /** * ssz_snappy encoding strategy writer. @@ -9,9 +8,7 @@ import snappy from "@chainsafe/snappy-stream"; * | * ``` */ -export async function* writeSszSnappyPayload(bodyData: Uint8Array): AsyncGenerator { - yield* encodeSszSnappy(bodyData as Buffer); -} +export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => AsyncGenerator; /** * Buffered Snappy writer @@ -22,17 +19,5 @@ export async function* encodeSszSnappy(bytes: Buffer): AsyncGenerator { // By first computing and writing the SSZ byte length, the SSZ encoder can then directly // write the chunk contents to the stream. Snappy writer compresses frame by frame - - /** - * Use sync version (default) for compress as it is almost 2x faster than async - * one and most payloads are "1 chunk" and 100kb payloads (which would mostly be - * big bellatrix blocks with transactions) are just 2 chunks - * - * To use async version (for e.g. on big payloads) instantiate the stream with - * `createCompressStream({asyncCompress: true})` - */ - const stream = snappy.createCompressStream(); - stream.write(bytes); - stream.end(); - yield* source(stream); + yield* encodeSnappy(bytes); } diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/common.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/common.ts new file mode 100644 index 000000000000..1077bcdcd4fb --- /dev/null +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/common.ts @@ -0,0 +1,9 @@ +export enum ChunkType { + IDENTIFIER = 0xff, + COMPRESSED = 0x00, + UNCOMPRESSED = 0x01, + PADDING = 0xfe, +} + +export const IDENTIFIER = Buffer.from([0x73, 0x4e, 0x61, 0x50, 0x70, 0x59]); +export const IDENTIFIER_FRAME = Buffer.from([0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59]); diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/compress.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/compress.ts new file mode 100644 index 000000000000..e1c3887eaa70 --- /dev/null +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/compress.ts @@ -0,0 +1,54 @@ +import snappy from "snappy"; +import crc32c from "@chainsafe/fast-crc32c"; +import {ChunkType, IDENTIFIER_FRAME} from "./common.js"; + +// The logic in this file is largely copied (in simplified form) from https://github.com/ChainSafe/node-snappy-stream/ + +/** + * As per the snappy framing format for streams, the size of any uncompressed chunk can be + * no longer than 65536 bytes. + * + * From: https://github.com/google/snappy/blob/main/framing_format.txt#L90:L92 + */ +const UNCOMPRESSED_CHUNK_SIZE = 65536; + +function checksum(value: Buffer): Buffer { + const x = crc32c.calculate(value); + const result = Buffer.allocUnsafe?.(4) ?? Buffer.alloc(4); + + // As defined in section 3 of https://github.com/google/snappy/blob/master/framing_format.txt + // And other implementations for reference: + // Go: https://github.com/golang/snappy/blob/2e65f85255dbc3072edf28d6b5b8efc472979f5a/snappy.go#L97 + // Python: https://github.com/andrix/python-snappy/blob/602e9c10d743f71bef0bac5e4c4dffa17340d7b3/snappy/snappy.py#L70 + // Mask the right hand to (32 - 17) = 15 bits -> 0x7fff, to keep correct 32 bit values. + // Shift the left hand with >>> for correct 32 bit intermediate result. + // Then final >>> 0 for 32 bits output + result.writeUInt32LE((((x >>> 15) | ((x & 0x7fff) << 17)) + 0xa282ead8) >>> 0, 0); + + return result; +} + +export async function* encodeSnappy(bytes: Buffer): AsyncGenerator { + yield IDENTIFIER_FRAME; + + for (let i = 0; i < bytes.length; i += UNCOMPRESSED_CHUNK_SIZE) { + const chunk = bytes.subarray(i, i + UNCOMPRESSED_CHUNK_SIZE); + const compressed = snappy.compressSync(chunk); + if (compressed.length < chunk.length) { + const size = compressed.length + 4; + yield Buffer.concat([ + Buffer.from([ChunkType.COMPRESSED, size, size >> 8, size >> 16]), + checksum(chunk), + compressed, + ]); + } else { + const size = chunk.length + 4; + yield Buffer.concat([ + // + Buffer.from([ChunkType.UNCOMPRESSED, size, size >> 8, size >> 16]), + checksum(chunk), + chunk, + ]); + } + } +} diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/uncompress.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/uncompress.ts index f676cf86efbc..e97260989082 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/uncompress.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/snappyFrames/uncompress.ts @@ -1,7 +1,6 @@ import {uncompress} from "snappyjs"; import {Uint8ArrayList} from "uint8arraylist"; - -const IDENTIFIER = Buffer.from([0x73, 0x4e, 0x61, 0x50, 0x70, 0x59]); +import {ChunkType, IDENTIFIER} from "./common.js"; export class SnappyFramesUncompress { private buffer = new Uint8ArrayList(); @@ -70,13 +69,6 @@ type UncompressState = { foundIdentifier: boolean; }; -enum ChunkType { - IDENTIFIER = 0xff, - COMPRESSED = 0x00, - UNCOMPRESSED = 0x01, - PADDING = 0xfe, -} - function getFrameSize(buffer: Uint8ArrayList, offset: number): number { return buffer.get(offset) + (buffer.get(offset + 1) << 8) + (buffer.get(offset + 2) << 16); } diff --git a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts index 6cdffef6e6ba..2abb99e35d54 100644 --- a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts +++ b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts @@ -1,45 +1,46 @@ import {expect} from "chai"; import {Uint8ArrayList} from "uint8arraylist"; -import snappy from "@chainsafe/snappy-stream"; +import {pipe} from "it-pipe"; import {SnappyFramesUncompress} from "../../../../../src/encodingStrategies/sszSnappy/snappyFrames/uncompress.js"; +import {encodeSnappy} from "../../../../../src/encodingStrategies/sszSnappy/snappyFrames/compress.js"; describe("encodingStrategies / sszSnappy / snappy frames / uncompress", function () { it("should work with short input", function (done) { - const compressStream = snappy.createCompressStream(); + const testData = "Small test data"; + const compressIterable = encodeSnappy(Buffer.from(testData)); const decompress = new SnappyFramesUncompress(); - const testData = "Small test data"; - - compressStream.on("data", function (data) { - const result = decompress.uncompress(data); - if (result) { - expect(result.subarray().toString()).to.be.equal(testData); - done(); + void pipe(compressIterable, async function (source) { + for await (const data of source) { + const result = decompress.uncompress(new Uint8ArrayList(data)); + if (result) { + expect(result.subarray().toString()).to.be.equal(testData); + done(); + } } }); - - compressStream.write(testData); }); it("should work with huge input", function (done) { - const compressStream = snappy.createCompressStream(); - - const decompress = new SnappyFramesUncompress(); - const testData = Buffer.alloc(100000, 4).toString(); + const compressIterable = encodeSnappy(Buffer.from(testData)); let result = Buffer.alloc(0); + const decompress = new SnappyFramesUncompress(); - compressStream.on("data", function (data) { - // testData will come compressed as two or more chunks - result = Buffer.concat([result, decompress.uncompress(data)?.subarray() ?? Buffer.alloc(0)]); - if (result.length === testData.length) { - expect(result.toString()).to.be.equal(testData); - done(); + void pipe(compressIterable, async function (source) { + for await (const data of source) { + // testData will come compressed as two or more chunks + result = Buffer.concat([ + result, + decompress.uncompress(new Uint8ArrayList(data))?.subarray() ?? Buffer.alloc(0), + ]); + if (result.length === testData.length) { + expect(result.toString()).to.be.equal(testData); + done(); + } } }); - - compressStream.write(testData); }); it("should detect malformed input", function () { diff --git a/packages/reqresp/test/utils/index.ts b/packages/reqresp/test/utils/index.ts index eb67a8cdf2c3..2ba3afd25976 100644 --- a/packages/reqresp/test/utils/index.ts +++ b/packages/reqresp/test/utils/index.ts @@ -19,7 +19,10 @@ export async function* arrToSource(arr: T[]): AsyncGenerator { * Wrapper for type-safety to ensure and array of Buffers is equal with a diff in hex */ export function expectEqualByteChunks(chunks: Uint8Array[], expectedChunks: Uint8Array[], message?: string): void { - expect(chunks.map(toHexString)).to.deep.equal(expectedChunks.map(toHexString), message); + expect(chunks.map(toHexString).join("").replace(/0x/g, "")).to.deep.equal( + expectedChunks.map(toHexString).join("").replace(/0x/g, ""), + message + ); } export function expectInEqualByteChunks(chunks: Uint8Array[], expectedChunks: Uint8Array[], message?: string): void { diff --git a/types/snappy-stream/index.d.ts b/types/snappy-stream/index.d.ts deleted file mode 100644 index c984683d5238..000000000000 --- a/types/snappy-stream/index.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -declare module "@chainsafe/snappy-stream" { - import {Transform} from "node:stream"; - - export function createUncompressStream(opts?: {asBuffer?: boolean}): Transform; - export function createCompressStream(opts?:{asyncCompress?: boolean}): Transform; -} diff --git a/types/stream-to-it/index.d.ts b/types/stream-to-it/index.d.ts deleted file mode 100644 index e384a748686f..000000000000 --- a/types/stream-to-it/index.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -declare module "stream-to-it" { - import {Readable} from "node:stream"; - export function source(readable: Readable): AsyncGenerator; -} diff --git a/yarn.lock b/yarn.lock index 99b2f2c97a42..9e287acf2c20 100644 --- a/yarn.lock +++ b/yarn.lock @@ -533,10 +533,10 @@ resolve "^1.10.1" semver "^6.1.0" -"@chainsafe/fast-crc32c@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@chainsafe/fast-crc32c/-/fast-crc32c-4.0.0.tgz#840ed6a1b140407e0521f4b1b5d54da534a6808d" - integrity sha512-N4gUwrL3yGgLgbXsCZObUh+RZw9aIA1yQ41yH/2V4MFYUj90LCTv8IfZmqIes5VLcEQ2kgPSs34OTmQRrG4PHQ== +"@chainsafe/fast-crc32c@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@chainsafe/fast-crc32c/-/fast-crc32c-4.1.1.tgz#f551284ecf8325f676a1e26b938bcca51b9c8d93" + integrity sha512-KgeCF52ciO6xHnlZAh/7azPM7zRQ0+Lism6P0MLE2jS2GnWL5NYjN7B8sEdxD7luFm/npt8qrdkaEMYNg5Apow== optionalDependencies: "@node-rs/crc32" "^1.6.0" @@ -627,18 +627,6 @@ resolved "https://registry.npmjs.org/@chainsafe/persistent-ts/-/persistent-ts-0.19.1.tgz" integrity sha512-fUFFFFxdcpYkMAHnjm83EYL/R/smtVmEkJr3FGSI6dwPk4ue9rXjEHf7FTd3V8AbVOcTJGriN4cYf2V+HOYkjQ== -"@chainsafe/snappy-stream@^5.1.2": - version "5.1.2" - resolved "https://registry.yarnpkg.com/@chainsafe/snappy-stream/-/snappy-stream-5.1.2.tgz#4374ed018b1d0bb6459f0cb4280df682bb97e903" - integrity sha512-zNhIzEL5k86y6qzXDF6nUIvHGON3SyQK7Vo63jvKyY5AlBJkupVDW8qnbQ9X0vfzw0w79VFYGfv7NXwtb2u4dg== - dependencies: - "@chainsafe/fast-crc32c" "^4.0.0" - bl "^1.0.0" - buffer-alloc "^1.2.0" - buffer-equal "1.0.0" - buffer-from "^1.1.1" - snappy "^6.3.5" - "@chainsafe/ssz@^0.10.2": version "0.10.2" resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.10.2.tgz#c782929e1bb25fec66ba72e75934b31fd087579e" @@ -2362,6 +2350,71 @@ uint8arrays "^4.0.2" varint "^6.0.0" +"@napi-rs/snappy-android-arm-eabi@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.2.2.tgz#85fee3ba198dad4b444b5f12bceebcf72db0d65e" + integrity sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw== + +"@napi-rs/snappy-android-arm64@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.2.2.tgz#386219c790c729aa0ced7ac6e3ac846892c5dd6d" + integrity sha512-2R/A3qok+nGtpVK8oUMcrIi5OMDckGYNoBLFyli3zp8w6IArPRfg1yOfVUcHvpUDTo9T7LOS1fXgMOoC796eQw== + +"@napi-rs/snappy-darwin-arm64@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.2.2.tgz#32bd351c695fbf60c899b365fff4f64bcd8b612c" + integrity sha512-USgArHbfrmdbuq33bD5ssbkPIoT7YCXCRLmZpDS6dMDrx+iM7eD2BecNbOOo7/v1eu6TRmQ0xOzeQ6I/9FIi5g== + +"@napi-rs/snappy-darwin-x64@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.2.2.tgz#71a8fca67a1fccb6323b8520d8d90c6b5da7c577" + integrity sha512-0APDu8iO5iT0IJKblk2lH0VpWSl9zOZndZKnBYIc+ei1npw2L5QvuErFOTeTdHBtzvUHASB+9bvgaWnQo4PvTQ== + +"@napi-rs/snappy-freebsd-x64@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.2.2.tgz#42102cbdaac39748520c9518c4fd9d3241d83a80" + integrity sha512-mRTCJsuzy0o/B0Hnp9CwNB5V6cOJ4wedDTWEthsdKHSsQlO7WU9W1yP7H3Qv3Ccp/ZfMyrmG98Ad7u7lG58WXA== + +"@napi-rs/snappy-linux-arm-gnueabihf@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.2.2.tgz#7e26ff0d974153c8b87160e99f60259ee9e14f0d" + integrity sha512-v1uzm8+6uYjasBPcFkv90VLZ+WhLzr/tnfkZ/iD9mHYiULqkqpRuC8zvc3FZaJy5wLQE9zTDkTJN1IvUcZ+Vcg== + +"@napi-rs/snappy-linux-arm64-gnu@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.2.2.tgz#992b2c4162da8d1da37ba2988700365975c21f3a" + integrity sha512-LrEMa5pBScs4GXWOn6ZYXfQ72IzoolZw5txqUHVGs8eK4g1HR9HTHhb2oY5ySNaKakG5sOgMsb1rwaEnjhChmQ== + +"@napi-rs/snappy-linux-arm64-musl@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.2.2.tgz#808dbf14b8789a8be7ecebef7606311f4df694fd" + integrity sha512-3orWZo9hUpGQcB+3aTLW7UFDqNCQfbr0+MvV67x8nMNYj5eAeUtMmUE/HxLznHO4eZ1qSqiTwLbVx05/Socdlw== + +"@napi-rs/snappy-linux-x64-gnu@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.2.2.tgz#681bfa25d8ed38a0bbec56827aa2762146c7e035" + integrity sha512-jZt8Jit/HHDcavt80zxEkDpH+R1Ic0ssiVCoueASzMXa7vwPJeF4ZxZyqUw4qeSy7n8UUExomu8G8ZbP6VKhgw== + +"@napi-rs/snappy-linux-x64-musl@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.2.2.tgz#4607f33fd0ef95a11143deff0d465428abcaae5a" + integrity sha512-Dh96IXgcZrV39a+Tej/owcd9vr5ihiZ3KRix11rr1v0MWtVb61+H1GXXlz6+Zcx9y8jM1NmOuiIuJwkV4vZ4WA== + +"@napi-rs/snappy-win32-arm64-msvc@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.2.2.tgz#88eb45cfdc66bb3c6b51f10903b29848f42a5c6d" + integrity sha512-9No0b3xGbHSWv2wtLEn3MO76Yopn1U2TdemZpCaEgOGccz1V+a/1d16Piz3ofSmnA13HGFz3h9NwZH9EOaIgYA== + +"@napi-rs/snappy-win32-ia32-msvc@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.2.2.tgz#e336064445e3c764bc3464640584d191c0fcf6dc" + integrity sha512-QiGe+0G86J74Qz1JcHtBwM3OYdTni1hX1PFyLRo3HhQUSpmi13Bzc1En7APn+6Pvo7gkrcy81dObGLDSxFAkQQ== + +"@napi-rs/snappy-win32-x64-msvc@7.2.2": + version "7.2.2" + resolved "https://registry.yarnpkg.com/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.2.2.tgz#4f598d3a5d50904d9f72433819f68b21eaec4f7d" + integrity sha512-a43cyx1nK0daw6BZxVcvDEXxKMFLSBSDTAhsFD0VqSKcC7MGUBMaqyoWUcMiI7LBSz4bxUmxDWKfCYzpEmeb3w== + "@noble/curves@1.0.0", "@noble/curves@~1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@noble/curves/-/curves-1.0.0.tgz#e40be8c7daf088aaf291887cbc73f43464a92932" @@ -4886,14 +4939,6 @@ bip39@^3.1.0: dependencies: "@noble/hashes" "^1.2.0" -bl@^1.0.0: - version "1.2.3" - resolved "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz" - integrity sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww== - dependencies: - readable-stream "^2.3.5" - safe-buffer "^5.1.1" - bl@^4.0.3, bl@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz" @@ -5135,44 +5180,16 @@ bs58check@^2.1.2: create-hash "^1.1.0" safe-buffer "^5.1.2" -buffer-alloc-unsafe@^1.1.0: - version "1.1.0" - resolved "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz" - integrity sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg== - -buffer-alloc@^1.2.0: - version "1.2.0" - resolved "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz" - integrity sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow== - dependencies: - buffer-alloc-unsafe "^1.1.0" - buffer-fill "^1.0.0" - buffer-crc32@^0.2.1, buffer-crc32@^0.2.13, buffer-crc32@~0.2.3: version "0.2.13" resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== -buffer-equal@1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz" - integrity sha1-WWFrSYME1Var1GaWayLu2j7KX74= - -buffer-fill@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz" - integrity sha1-+PeLdniYiO858gXNY39o5wISKyw= - buffer-from@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -buffer-from@^1.1.1: - version "1.1.1" - resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz" - integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== - buffer-to-arraybuffer@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/buffer-to-arraybuffer/-/buffer-to-arraybuffer-0.0.5.tgz#6064a40fa76eb43c723aba9ef8f6e1216d10511a" @@ -5551,7 +5568,7 @@ chokidar@3.5.3, chokidar@^3.5.1: optionalDependencies: fsevents "~2.3.2" -chownr@^1.0.1, chownr@^1.1.1, chownr@^1.1.4: +chownr@^1.1.1, chownr@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== @@ -6487,7 +6504,7 @@ detect-indent@^5.0.0: resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-5.0.0.tgz#3871cc0a6a002e8c3e5b3cf7f336264675f06b9d" integrity sha512-rlpvsxUtM0PQvy9iZe640/IWwWYyBsTApREbA1pHOpmOUIl9MkP/U4z7vTtg4Oaojvqhxt7sdufnT0EzGaR31g== -detect-libc@^1.0.2, detect-libc@^1.0.3: +detect-libc@^1.0.2: version "1.0.3" resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz" integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups= @@ -6731,7 +6748,7 @@ encoding@^0.1.12, encoding@^0.1.13: dependencies: iconv-lite "^0.6.2" -end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1: +end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.4" resolved "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz" integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== @@ -7496,11 +7513,6 @@ execa@^5.0.0: signal-exit "^3.0.3" strip-final-newline "^2.0.0" -expand-template@^2.0.3: - version "2.0.3" - resolved "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz" - integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== - expand-tilde@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz" @@ -8336,11 +8348,6 @@ gitconfiglocal@^1.0.0: dependencies: ini "^1.3.2" -github-from-package@0.0.0: - version "0.0.0" - resolved "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz" - integrity sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4= - glob-parent@5.1.2, glob-parent@^5.1.2, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -11252,11 +11259,6 @@ nan@^2.13.2: resolved "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz" integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== -nan@^2.14.1: - version "2.14.1" - resolved "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz" - integrity sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw== - nan@^2.15.0, nan@^2.16.0: version "2.16.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.16.0.tgz#664f43e45460fb98faf00edca0bb0d7b8dce7916" @@ -11277,11 +11279,6 @@ nanoid@^4.0.0: resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-4.0.2.tgz#140b3c5003959adbebf521c170f282c5e7f9fb9e" integrity sha512-7ZtY5KTCNheRGfEFxnedV5zFiORN1+Y1N6zvPTnHQd8ENUvfaDBeuJDZb2bN/oXwXxu3qkTXDzy57W5vAmDTBw== -napi-build-utils@^1.0.1: - version "1.0.2" - resolved "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz" - integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg== - napi-macros@~2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz" @@ -11347,13 +11344,6 @@ nise@^5.1.4: just-extend "^4.0.2" path-to-regexp "^1.7.0" -node-abi@^2.7.0: - version "2.26.0" - resolved "https://registry.npmjs.org/node-abi/-/node-abi-2.26.0.tgz" - integrity sha512-ag/Vos/mXXpWLLAYWsAoQdgS+gW7IwvgMLOgqopm/DbzAjazLltzgzpVMsFlgmo9TzG5hGXeaBZx2AI731RIsQ== - dependencies: - semver "^5.4.1" - node-addon-api@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-2.0.2.tgz#432cfa82962ce494b132e9d72a15b29f71ff5d32" @@ -11501,11 +11491,6 @@ node-releases@^2.0.6: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.6.tgz#8a7088c63a55e493845683ebf3c828d8c51c5503" integrity sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg== -noop-logger@^0.1.1: - version "0.1.1" - resolved "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz" - integrity sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI= - nopt@^4.0.1: version "4.0.3" resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48" @@ -11804,7 +11789,7 @@ npmlog@6.0.2, npmlog@^6.0.2: gauge "^4.0.3" set-blocking "^2.0.0" -npmlog@^4.0.1, npmlog@^4.0.2: +npmlog@^4.0.2: version "4.1.2" resolved "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz" integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg== @@ -12116,7 +12101,7 @@ os-browserify@^0.3.0: resolved "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz" integrity sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc= -os-homedir@^1.0.0, os-homedir@^1.0.1: +os-homedir@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz" integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= @@ -12656,28 +12641,6 @@ postcss-selector-parser@^6.0.10: cssesc "^3.0.0" util-deprecate "^1.0.2" -prebuild-install@5.3.0: - version "5.3.0" - resolved "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.0.tgz" - integrity sha512-aaLVANlj4HgZweKttFNUVNRxDukytuIuxeK2boIMHjagNJCiVKWFsKF4tCE3ql3GbrD2tExPQ7/pwtEJcHNZeg== - dependencies: - detect-libc "^1.0.3" - expand-template "^2.0.3" - github-from-package "0.0.0" - minimist "^1.2.0" - mkdirp "^0.5.1" - napi-build-utils "^1.0.1" - node-abi "^2.7.0" - noop-logger "^0.1.1" - npmlog "^4.0.1" - os-homedir "^1.0.1" - pump "^2.0.1" - rc "^1.2.7" - simple-get "^2.7.0" - tar-fs "^1.13.0" - tunnel-agent "^0.6.0" - which-pm-runs "^1.0.0" - prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz" @@ -12918,22 +12881,6 @@ public-encrypt@^4.0.0: randombytes "^2.0.1" safe-buffer "^5.1.2" -pump@^1.0.0: - version "1.0.3" - resolved "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz" - integrity sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw== - dependencies: - end-of-stream "^1.1.0" - once "^1.3.1" - -pump@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz" - integrity sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA== - dependencies: - end-of-stream "^1.1.0" - once "^1.3.1" - pump@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz" @@ -13215,7 +13162,7 @@ readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.2, readable-stre string_decoder "^1.1.1" util-deprecate "^1.0.1" -readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.3.0, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@^2.3.7, readable-stream@~2.3.6: +readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@^2.3.7, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -13889,14 +13836,24 @@ smart-buffer@^4.2.0: resolved "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz" integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== -snappy@^6.3.5: - version "6.3.5" - resolved "https://registry.npmjs.org/snappy/-/snappy-6.3.5.tgz" - integrity sha512-lonrUtdp1b1uDn1dbwgQbBsb5BbaiLeKq+AGwOk2No+en+VvJThwmtztwulEQsLinRF681pBqib0NUZaizKLIA== - dependencies: - bindings "^1.3.1" - nan "^2.14.1" - prebuild-install "5.3.0" +snappy@^7.2.2: + version "7.2.2" + resolved "https://registry.yarnpkg.com/snappy/-/snappy-7.2.2.tgz#dbd9217ae06b651c073856036618c2dc8992ef17" + integrity sha512-iADMq1kY0v3vJmGTuKcFWSXt15qYUz7wFkArOrsSg0IFfI3nJqIJvK2/ZbEIndg7erIJLtAVX2nSOqPz7DcwbA== + optionalDependencies: + "@napi-rs/snappy-android-arm-eabi" "7.2.2" + "@napi-rs/snappy-android-arm64" "7.2.2" + "@napi-rs/snappy-darwin-arm64" "7.2.2" + "@napi-rs/snappy-darwin-x64" "7.2.2" + "@napi-rs/snappy-freebsd-x64" "7.2.2" + "@napi-rs/snappy-linux-arm-gnueabihf" "7.2.2" + "@napi-rs/snappy-linux-arm64-gnu" "7.2.2" + "@napi-rs/snappy-linux-arm64-musl" "7.2.2" + "@napi-rs/snappy-linux-x64-gnu" "7.2.2" + "@napi-rs/snappy-linux-x64-musl" "7.2.2" + "@napi-rs/snappy-win32-arm64-msvc" "7.2.2" + "@napi-rs/snappy-win32-ia32-msvc" "7.2.2" + "@napi-rs/snappy-win32-x64-msvc" "7.2.2" snappyjs@^0.7.0: version "0.7.0" @@ -14187,13 +14144,6 @@ stream-to-it@^0.2.2: dependencies: get-iterator "^1.0.2" -stream-to-it@^0.2.4: - version "0.2.4" - resolved "https://registry.yarnpkg.com/stream-to-it/-/stream-to-it-0.2.4.tgz#d2fd7bfbd4a899b4c0d6a7e6a533723af5749bd0" - integrity sha512-4vEbkSs83OahpmBybNJXlJd7d6/RxzkkSdT3I0mnGt79Xd2Kk+e1JqbvAvsQfCeKj3aKb0QIWkyK3/n0j506vQ== - dependencies: - get-iterator "^1.0.2" - streamroller@^3.1.1: version "3.1.2" resolved "https://registry.yarnpkg.com/streamroller/-/streamroller-3.1.2.tgz#abd444560768b340f696307cf84d3f46e86c0e63" @@ -14466,16 +14416,6 @@ tapable@^2.1.1, tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -tar-fs@^1.13.0: - version "1.16.3" - resolved "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz" - integrity sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw== - dependencies: - chownr "^1.0.1" - mkdirp "^0.5.1" - pump "^1.0.0" - tar-stream "^1.1.2" - tar-fs@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" @@ -14496,19 +14436,6 @@ tar-fs@~2.0.1: pump "^3.0.0" tar-stream "^2.0.0" -tar-stream@^1.1.2: - version "1.6.2" - resolved "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz" - integrity sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A== - dependencies: - bl "^1.0.0" - buffer-alloc "^1.2.0" - end-of-stream "^1.0.0" - fs-constants "^1.0.0" - readable-stream "^2.3.0" - to-buffer "^1.1.1" - xtend "^4.0.0" - tar-stream@^2.0.0, tar-stream@^2.1.4, tar-stream@^2.2.0, tar-stream@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" @@ -14743,11 +14670,6 @@ to-arraybuffer@^1.0.0: resolved "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz" integrity sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M= -to-buffer@^1.1.1: - version "1.1.1" - resolved "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz" - integrity sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg== - to-fast-properties@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz" @@ -15760,11 +15682,6 @@ which-module@^2.0.0: resolved "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz" integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho= -which-pm-runs@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz" - integrity sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs= - which-typed-array@^1.1.2: version "1.1.4" resolved "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.4.tgz" From b23dca0368cfacf731a2a5837af97fa164d82bb4 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 21 Jun 2023 17:14:04 +0200 Subject: [PATCH 14/96] refactor: rename CLI flag to enable doppelganger protection (#5676) --- .../e2e/doppelganger/doppelganger.test.ts | 28 +++++++++---------- .../beacon-node/test/utils/node/validator.ts | 6 ++-- packages/cli/src/cmds/validator/handler.ts | 4 +-- packages/cli/src/cmds/validator/options.ts | 5 ++-- packages/validator/src/validator.ts | 4 +-- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts b/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts index 3241a1fbe6a8..0dec0192eed2 100644 --- a/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts +++ b/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts @@ -41,7 +41,7 @@ describe.skip("doppelganger / doppelganger test", function () { type TestConfig = { genesisTime?: number; - doppelgangerProtectionEnabled?: boolean; + doppelgangerProtection?: boolean; }; async function createBNAndVC(config?: TestConfig): Promise<{beaconNode: BeaconNode; validators: Validator[]}> { @@ -69,7 +69,7 @@ describe.skip("doppelganger / doppelganger test", function () { startIndex: 0, useRestApi: false, testLoggerOpts, - doppelgangerProtectionEnabled: config?.doppelgangerProtectionEnabled, + doppelgangerProtection: config?.doppelgangerProtection, }); afterEachCallbacks.push(() => Promise.all(validatorsWithDoppelganger.map((v) => v.close()))); @@ -83,7 +83,7 @@ describe.skip("doppelganger / doppelganger test", function () { const validatorIndex = 0; const {beaconNode: bn, validators: validatorsWithDoppelganger} = await createBNAndVC({ - doppelgangerProtectionEnabled: true, + doppelgangerProtection: true, }); const validatorUnderTest = validatorsWithDoppelganger[0]; @@ -118,7 +118,7 @@ describe.skip("doppelganger / doppelganger test", function () { const {beaconNode: bn, validators: validatorsWithDoppelganger} = await createBNAndVC({ genesisTime, - doppelgangerProtectionEnabled: true, + doppelgangerProtection: true, }); const {beaconNode: bn2, validators: validators} = await createBNAndVC({ @@ -148,7 +148,7 @@ describe.skip("doppelganger / doppelganger test", function () { it("should shut down validator if same key is active with same BN and started after genesis", async function () { this.timeout("10 min"); - const doppelgangerProtectionEnabled = true; + const doppelgangerProtection = true; const testLoggerOpts: TestLoggerOpts = {level: LogLevel.info}; // set genesis time to allow at least an epoch @@ -156,7 +156,7 @@ describe.skip("doppelganger / doppelganger test", function () { const {beaconNode: bn, validators: validator0WithDoppelganger} = await createBNAndVC({ genesisTime, - doppelgangerProtectionEnabled, + doppelgangerProtection, }); const {validators: validator0WithoutDoppelganger} = await getAndInitDevValidators({ @@ -166,7 +166,7 @@ describe.skip("doppelganger / doppelganger test", function () { startIndex: 0, useRestApi: false, testLoggerOpts, - doppelgangerProtectionEnabled: false, + doppelgangerProtection: false, }); afterEachCallbacks.push(() => Promise.all(validator0WithoutDoppelganger.map((v) => v.close()))); @@ -194,15 +194,15 @@ describe.skip("doppelganger / doppelganger test", function () { it("should not shut down validator if key is different", async function () { this.timeout("10 min"); - const doppelgangerProtectionEnabled = true; + const doppelgangerProtection = true; const {beaconNode: bn, validators: validatorsWithDoppelganger} = await createBNAndVC({ - doppelgangerProtectionEnabled, + doppelgangerProtection, }); const {beaconNode: bn2, validators: validators} = await createBNAndVC({ genesisTime: bn.chain.getHeadState().genesisTime, - doppelgangerProtectionEnabled: false, + doppelgangerProtection: false, }); await connect(bn2.network, bn.network); @@ -226,14 +226,14 @@ describe.skip("doppelganger / doppelganger test", function () { it("should not sign block if doppelganger period has not passed and not started at genesis", async function () { this.timeout("10 min"); - const doppelgangerProtectionEnabled = true; + const doppelgangerProtection = true; // set genesis time to allow at least an epoch const genesisTime = Math.floor(Date.now() / 1000) - SLOTS_PER_EPOCH * beaconParams.SECONDS_PER_SLOT; const {beaconNode: bn, validators: validatorsWithDoppelganger} = await createBNAndVC({ genesisTime, - doppelgangerProtectionEnabled, + doppelgangerProtection, }); const validatorUnderTest = validatorsWithDoppelganger[0]; @@ -259,14 +259,14 @@ describe.skip("doppelganger / doppelganger test", function () { it("should not sign attestations if doppelganger period has not passed and started after genesis", async function () { this.timeout("10 min"); - const doppelgangerProtectionEnabled = true; + const doppelgangerProtection = true; // set genesis time to allow at least an epoch const genesisTime = Math.floor(Date.now() / 1000) - SLOTS_PER_EPOCH * beaconParams.SECONDS_PER_SLOT; const {beaconNode: bn, validators: validatorsWithDoppelganger} = await createBNAndVC({ genesisTime, - doppelgangerProtectionEnabled, + doppelgangerProtection, }); const validatorUnderTest = validatorsWithDoppelganger[0]; diff --git a/packages/beacon-node/test/utils/node/validator.ts b/packages/beacon-node/test/utils/node/validator.ts index eeaf038d58cc..34acd9982572 100644 --- a/packages/beacon-node/test/utils/node/validator.ts +++ b/packages/beacon-node/test/utils/node/validator.ts @@ -16,7 +16,7 @@ export async function getAndInitDevValidators({ useRestApi, testLoggerOpts, externalSignerUrl, - doppelgangerProtectionEnabled = false, + doppelgangerProtection = false, valProposerConfig, }: { node: BeaconNode; @@ -26,7 +26,7 @@ export async function getAndInitDevValidators({ useRestApi?: boolean; testLoggerOpts?: TestLoggerOpts; externalSignerUrl?: string; - doppelgangerProtectionEnabled?: boolean; + doppelgangerProtection?: boolean; valProposerConfig?: ValidatorProposerConfig; }): Promise<{validators: Validator[]; secretKeys: SecretKey[]}> { const validators: Promise[] = []; @@ -70,7 +70,7 @@ export async function getAndInitDevValidators({ processShutdownCallback: () => {}, abortController, signers, - doppelgangerProtectionEnabled, + doppelgangerProtection, valProposerConfig, }) ); diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index c0f8e6babdc2..a9f60a422bd5 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -36,7 +36,7 @@ import {KeymanagerRestApiServer} from "./keymanager/server.js"; export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Promise { const {config, network} = getBeaconConfigFromArgs(args); - const doppelgangerProtectionEnabled = args.doppelgangerProtectionEnabled; + const {doppelgangerProtection} = args; const validatorPaths = getValidatorPaths(args, network); const accountPaths = getAccountPaths(args, network); @@ -160,7 +160,7 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr processShutdownCallback, signers, abortController, - doppelgangerProtectionEnabled, + doppelgangerProtection, afterBlockDelaySlotFraction: args.afterBlockDelaySlotFraction, scAfterBlockDelaySlotFraction: args.scAfterBlockDelaySlotFraction, disableAttestationGrouping: args.disableAttestationGrouping, diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index 2b6fb28ab16d..117c7d5867c0 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -40,7 +40,7 @@ export type IValidatorCliArgs = AccountValidatorArgs & suggestedFeeRecipient?: string; proposerSettingsFile?: string; strictFeeRecipientCheck?: boolean; - doppelgangerProtectionEnabled?: boolean; + doppelgangerProtection?: boolean; defaultGasLimit?: number; builder?: boolean; @@ -256,7 +256,8 @@ export const validatorOptions: CliCommandOptions = { type: "string", }, - doppelgangerProtectionEnabled: { + doppelgangerProtection: { + alias: ["doppelgangerProtectionEnabled"], description: "Enables Doppelganger protection", default: false, type: "boolean", diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index e2aa84d2bb87..791cf6fff51f 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -34,7 +34,7 @@ export type ValidatorOptions = { afterBlockDelaySlotFraction?: number; scAfterBlockDelaySlotFraction?: number; disableAttestationGrouping?: boolean; - doppelgangerProtectionEnabled?: boolean; + doppelgangerProtection?: boolean; closed?: boolean; valProposerConfig?: ValidatorProposerConfig; distributed?: boolean; @@ -93,7 +93,7 @@ export class Validator { } const indicesService = new IndicesService(logger, api, metrics); - const doppelgangerService = opts.doppelgangerProtectionEnabled + const doppelgangerService = opts.doppelgangerProtection ? new DoppelgangerService(logger, clock, api, indicesService, opts.processShutdownCallback, metrics) : null; From 2a2c517456f8173dd81216b1c45757793d53a3ac Mon Sep 17 00:00:00 2001 From: Cayman Date: Wed, 21 Jun 2023 12:15:37 -0400 Subject: [PATCH 15/96] chore: update cpu-features subdependency (#5674) --- yarn.lock | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/yarn.lock b/yarn.lock index 9e287acf2c20..db9ee4d9e276 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4910,7 +4910,7 @@ binary-extensions@^2.0.0: resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== -bindings@^1.3.0, bindings@^1.3.1, bindings@^1.5.0: +bindings@^1.3.0, bindings@^1.5.0: version "1.5.0" resolved "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz" integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== @@ -5244,10 +5244,10 @@ bufio@~1.0.7: resolved "https://registry.npmjs.org/bufio/-/bufio-1.0.7.tgz" integrity sha512-bd1dDQhiC+bEbEfg56IdBv7faWa6OipMs/AFFFvtFnB3wAYjlwQpQRZ0pm6ZkgtfL0pILRXhKxOiQj6UzoMR7A== -buildcheck@0.0.3: - version "0.0.3" - resolved "https://registry.yarnpkg.com/buildcheck/-/buildcheck-0.0.3.tgz#70451897a95d80f7807e68fc412eb2e7e35ff4d5" - integrity sha512-pziaA+p/wdVImfcbsZLNF32EiWyujlQLwolMqUQE8xpKNOH7KmZQaY8sXN7DGOEzPAElo9QTaeNRfGnf3iOJbA== +buildcheck@~0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/buildcheck/-/buildcheck-0.0.6.tgz#89aa6e417cfd1e2196e3f8fe915eb709d2fe4238" + integrity sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A== builtin-status-codes@^3.0.0: version "3.0.0" @@ -6119,12 +6119,12 @@ cosmiconfig@7.0.0: yaml "^1.10.0" cpu-features@~0.0.4: - version "0.0.4" - resolved "https://registry.yarnpkg.com/cpu-features/-/cpu-features-0.0.4.tgz#0023475bb4f4c525869c162e4108099e35bf19d8" - integrity sha512-fKiZ/zp1mUwQbnzb9IghXtHtDoTMtNeb8oYGx6kX2SYfhnG0HNdBEBIzB9b5KlXu5DQPhfy3mInbBxFcgwAr3A== + version "0.0.8" + resolved "https://registry.yarnpkg.com/cpu-features/-/cpu-features-0.0.8.tgz#a2d464b023b8ad09004c8cdca23b33f192f63546" + integrity sha512-BbHBvtYhUhksqTjr6bhNOjGgMnhwhGTQmOoZGD+K7BCaQDCuZl/Ve1ZxUSMRwVC4D/rkCPQ2MAIeYzrWyK7eEg== dependencies: - buildcheck "0.0.3" - nan "^2.15.0" + buildcheck "~0.0.6" + nan "^2.17.0" crc-32@^1.2.0: version "1.2.2" @@ -11254,15 +11254,10 @@ mute-stream@1.0.0: resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-1.0.0.tgz#e31bd9fe62f0aed23520aa4324ea6671531e013e" integrity sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA== -nan@^2.13.2: - version "2.14.2" - resolved "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz" - integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== - -nan@^2.15.0, nan@^2.16.0: - version "2.16.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.16.0.tgz#664f43e45460fb98faf00edca0bb0d7b8dce7916" - integrity sha512-UdAqHyFngu7TfQKsCBgAA6pWDkT8MAO7d0jyOecVhN5354xbLqdn8mV9Tat9gepAupm0bt2DbeaSC8vS52MuFA== +nan@^2.13.2, nan@^2.16.0, nan@^2.17.0: + version "2.17.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.17.0.tgz#c0150a2368a182f033e9aa5195ec76ea41a199cb" + integrity sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ== nano-json-stream-parser@^0.1.2: version "0.1.2" From 9e57b7d320fcb2440cb8a8215e24a9232cf9fc0a Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 21 Jun 2023 20:24:27 +0200 Subject: [PATCH 16/96] ci: check docs spelling (#5527) * Check docs spelling * Add config * Fix docs spelling * Lint docs format * docs(flamegraph): remove note about script that was removed * sort wordlist * Set sort algo * Fix CI pipeline * hope to fix CI ordering * Disable wordlist sort check * docs(CONTRIBUTING): remove merge conflict marker and add codeblock types * refactor: change to pyspelling.yml for local runs * docs: make spell-checking wordlist case insensitive * fix: sort .wordlist.txt * refactor: alphabetize script order * fix: add docs/reference to gitignore * docs: add codeblocks to pyspelling ignore and clear all errors * docs: ignore possessive endings in spellcheck * docs: clear spelling errors in package readmes * docs: pyspelling fix generated cli.md * feat(workflows): lint generated docs * feat(workflows): add word sort to docs-check * test: unsort wordlist * fix: sort wordlist * refactor(workflow): remove unused comments --------- Co-authored-by: matthewkeil Co-authored-by: Cayman --- .github/workflows/docs-check.yml | 50 +++++++ .github/workflows/docs.yml | 5 + .github/workflows/test.yml | 7 +- .gitignore | 5 +- .prettierignore | 4 +- .pyspelling.yml | 42 ++++++ .wordlist.txt | 130 ++++++++++++++++++ CONTRIBUTING.md | 74 +++------- README.md | 32 ++--- RELEASE.md | 6 +- docs/design/depgraph.md | 32 ++--- docs/index.md | 2 +- docs/install/docker.md | 7 +- docs/install/npm.md | 2 +- docs/install/source.md | 4 +- docs/libraries/index.md | 30 ++-- docs/quickstart.md | 14 +- docs/tools/flamegraphs.md | 28 ++-- docs/usage/beacon-management.md | 24 +++- docs/usage/client-monitoring.md | 4 +- docs/usage/local.md | 11 +- docs/usage/mev-integration.md | 22 +-- docs/usage/prometheus-grafana.md | 4 +- docs/usage/validator-management.md | 18 ++- package.json | 6 +- packages/api/README.md | 2 +- .../scripts/el-interop/gethdocker/README.md | 1 + packages/cli/README.md | 2 +- packages/cli/docsgen/index.ts | 2 +- packages/cli/src/cmds/beacon/options.ts | 2 +- .../cmds/validator/blsToExecutionChange.ts | 2 +- packages/cli/src/cmds/validator/list.ts | 2 +- packages/cli/src/cmds/validator/options.ts | 30 ++-- .../validator/slashingProtection/export.ts | 2 +- .../validator/slashingProtection/options.ts | 2 +- .../cli/src/cmds/validator/voluntaryExit.ts | 4 +- .../src/options/beaconNodeOptions/builder.ts | 2 +- .../src/options/beaconNodeOptions/chain.ts | 4 +- .../cli/src/options/beaconNodeOptions/eth1.ts | 2 +- packages/cli/src/options/logOptions.ts | 4 +- packages/params/README.md | 5 +- packages/prover/README.md | 2 +- packages/spec-test-util/README.md | 2 +- packages/types/README.md | 8 +- scripts/wordlist_sort.sh | 7 + scripts/wordlist_sort_check.sh | 23 ++++ yarn.lock | 49 +++++++ 47 files changed, 505 insertions(+), 217 deletions(-) create mode 100644 .github/workflows/docs-check.yml create mode 100644 .pyspelling.yml create mode 100644 .wordlist.txt create mode 100755 scripts/wordlist_sort.sh create mode 100755 scripts/wordlist_sort_check.sh diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml new file mode 100644 index 000000000000..bad86bd201b9 --- /dev/null +++ b/.github/workflows/docs-check.yml @@ -0,0 +1,50 @@ +name: Check docs + +on: + push: + # We intentionally don't run push on feature branches. See PR for rational. + branches: [unstable, stable] + pull_request: + +jobs: + build: + name: Docs spellcheck + runs-on: ubuntu-latest + steps: + # - Uses YAML anchors in the future + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - name: Node.js version + id: node + run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT + - name: Restore dependencies + uses: actions/cache@master + id: cache-deps + with: + path: | + node_modules + packages/*/node_modules + key: ${{ runner.os }}-${{ steps.node.outputs.v8CppApiVersion }}-${{ hashFiles('**/yarn.lock', '**/package.json') }} + - name: Install & build + if: steps.cache-deps.outputs.cache-hit != 'true' + run: yarn install --frozen-lockfile && yarn build + - name: Build + run: yarn build + if: steps.cache-deps.outputs.cache-hit == 'true' + + - name: Check wordlist is sorted + run: scripts/wordlist_sort_check.sh + + - name: Build and collect docs + run: yarn build:docs + + # Run prettier check with fix after generating the docs. The CLI reference is formatted with prettier for + # deployed version so this will fail if not "fixable" + - name: Check docs format + run: yarn lint-docs:fix + + # Run spellcheck AFTER building docs, in case the CLI reference has issues + - name: Spellcheck + uses: rojopolis/spellcheck-github-actions@0.32.0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8f45529bc74a..3dacadfac3ec 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -36,12 +36,17 @@ jobs: - name: Build and collect docs run: yarn build:docs + - name: Lint built docs + run: yarn lint-docs:fix + - name: Set up Python uses: actions/setup-python@v1 + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r docs/requirements.txt + - name: Build docs run: mkdocs build --site-dir site -v --clean diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fa821219a851..52d224407de8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,13 +67,10 @@ jobs: run: scripts/assert_eslintrc_sorted.mjs - name: Check Types - run: yarn run check-types - # Test docs generation, even if not published - - name: Build docs - run: yarn build:docs + run: yarn check-types - name: README check - run: yarn run check-readme + run: yarn check-readme - name: Lint run: yarn lint diff --git a/.gitignore b/.gitignore index 3a8a7843b710..ce1ec6074979 100644 --- a/.gitignore +++ b/.gitignore @@ -40,10 +40,10 @@ packages/api/oapi-schemas # Autogenerated docs packages/**/docs packages/**/typedocs +docs/assets docs/packages +docs/reference docs/contributing.md -docs/assets -docs/reference/cli.md /site # Testnet artifacts @@ -70,5 +70,6 @@ packages/cli/.git-data.json # Build artifacts .last_build_unixsec +dictionary.dic temp/ diff --git a/.prettierignore b/.prettierignore index 68a34fe1a083..7871bd5d3ad7 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,5 @@ **/lib **/.nyc_output -/packages/*/spec-tests \ No newline at end of file +/packages/*/spec-tests +node_modules +**/node_modules \ No newline at end of file diff --git a/.pyspelling.yml b/.pyspelling.yml new file mode 100644 index 000000000000..6fb66c37f7e7 --- /dev/null +++ b/.pyspelling.yml @@ -0,0 +1,42 @@ +matrix: + - name: markdown + aspell: + lang: en + ignore-case: true + dictionary: + wordlists: + - .wordlist.txt + pipeline: + - pyspelling.filters.url: + - pyspelling.filters.markdown: + markdown_extensions: + - pymdownx.superfences: + - pymdownx.highlight: + - pymdownx.striphtml: + - pymdownx.magiclink: + - pyspelling.filters.url: + - pyspelling.filters.html: + comments: false + ignores: + - code + - pre + - pyspelling.filters.context: + context_visible_first: true + delimiters: + # Ignore possessive endings + - open: '(?<=\w)''s(?!\w)' + close: '\b' + # Ignore eth methods (e.g. eth_estimateGas) + - open: '(?:\s)eth_(?:\w*)' + close: '\s' + # Ignore flags in cli.md + - open: '--(?:\w*)' + close: '[^\w]' + # Ignore hex strings + - open: '0x[a-fA-F0-9]' + close: '[^a-fA-F0-9]' + sources: + - "docs/**/*.md" + - "CONTRIBUTING.md" + - "README.md" + - "packages/*/README.md" \ No newline at end of file diff --git a/.wordlist.txt b/.wordlist.txt new file mode 100644 index 000000000000..fbc71353e180 --- /dev/null +++ b/.wordlist.txt @@ -0,0 +1,130 @@ +APIs +AssemblyScript +BLS +BeaconNode +Besu +CLA +CLI +CTRL +Chai +ChainSafe +Customizations +Discv +DockerHub +Dockerized +EIP +EIPs +EL +ENR +ENRs +ESLint +ETH +Erigon +EthStaker +Ethereum +FX +Flamegraph +Flamegraphs +Github +Gossipsub +Grafana +HackMD +IPv +Infura +JSON +JWT +LGPL +LGPLv +LTS +Lerna +MEV +MacOS +Monorepo +NPM +NVM +Nethermind +NodeJS +NodeSource +PR +PRs +Plaintext +PoS +Quickstart +RPC +SHA +SSD +SSZ +Stakehouse +TOC +TTD +TypeScript +UI +UTF +VM +Vitalik +Wagyu +api +async +beaconcha +bootnode +bootnodes +chainConfig +chainsafe +cli +cmd +config +configs +const +constantish +cors +cryptographic +dApp +dApps +decrypt +deserialization +devnet +devnets +enum +envs +flamegraph +flamegraphs +goerli +interop +keypair +keystore +keystores +lightclient +linter +lockfile +mainnet +mdns +merkle +merkleization +monorepo +namespace +namespaced +namespaces +nodemodule +overriden +params +plaintext +prover +req +reqresp +runtime +sharding +ssz +stakers +subnets +tcp +testnet +testnets +todo +typesafe +udp +util +utils +validator +validators +wip +yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d6bb91c0c3c7..6be558ad7c5b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,6 +25,13 @@ To run tests: - :test_tube: Run `yarn check-types` to check TypeScript types. - :test_tube: Run `yarn lint` to run the linter (ESLint). +Contributing to tests: + +- Test must not depend on external live resources, such that running tests for a commit must be deterministic: + - Do not pull data from external APIs like execution JSON RPC (instead run a local node). + - Do not pull unpinned versions from DockerHub (use deterministic tag) or Github (checkout commit not branch). + - Carefully design tests that depend on timing sensitive events like p2p network e2e tests. Consider that Github runners are significantly less powerful than your development environment. + ### Debugging Spec Tests - To fix errors always focus on passing all minimal tests first without running mainnet tests. @@ -32,7 +39,7 @@ To run tests: - A single logical error can cause many spec tests to fail. To focus on a single test at a time you can use mocha's option `--bail` to stop at the first failed test - To then run only that failed test you can run against a specific file as use mocha's option `--grep` to run only one case -``` +```sh LODESTAR_PRESET=minimal ../../node_modules/.bin/mocha --config .mocharc.spec.yml test/spec/phase0/sanity.test.ts --inline-diffs --bail --grep "attestation" ``` @@ -40,29 +47,29 @@ LODESTAR_PRESET=minimal ../../node_modules/.bin/mocha --config .mocharc.spec.yml The docker-compose file requires that a `.env` file be present in this directory. The `default.env` file provides a template and can be copied `.env`: -``` +```sh cp default.env .env ``` -###### Beacon node only: +###### Beacon node only -``` +```sh docker-compose up -d ``` -###### Beacon node and validator: +###### Beacon node and validator -First, you must have keystores and their secrets available locally at `./keystores` and your password.txt in `./secrets` +First, you must have keystores and their secrets available locally at `./keystores` and your `password.txt` in `./secrets` -``` +```sh docker-compose -f docker-compose.yml -f docker-compose.validator.yml up -d ``` -###### Dockerized metrics + local beacon node: +###### Dockerized metrics + local beacon node Run a local beacon with `--metrics` enabled. Then start Prometheus + Grafana with all dashboards in `./dashboards` automatically loaded running: -``` +```sh ./docker/docker-compose.local_dev.sh ``` @@ -92,7 +99,7 @@ Unsure where to begin contributing to Lodestar? Here are some ideas! **Branch Naming** -If you are contributing from this repo prefix the branch name with your Github username (i.e. `myusername/short-description`) +If you are contributing from this repository prefix the branch name with your Github username (i.e. `myusername/short-description`) **Pull Request Naming** @@ -118,7 +125,7 @@ For example: ## Lodestar Monorepo -We're currently experimenting with hosting the majority of lodestar packages and support packages in this repository as a [monorepo](https://en.wikipedia.org/wiki/Monorepo). We're using [Lerna](https://lerna.js.org/) to manage the packages. See [packages/](https://github.com/ChainSafe/lodestar/tree/unstable/packages) for a list of packages hosted in this repo. +We're currently experimenting with hosting the majority of lodestar packages and support packages in this repository as a [monorepo](https://en.wikipedia.org/wiki/Monorepo). We're using [Lerna](https://lerna.js.org/) to manage the packages. See [packages/](https://github.com/ChainSafe/lodestar/tree/unstable/packages) for a list of packages hosted in this repository. ## Style Guide @@ -134,7 +141,7 @@ We're currently experimenting with hosting the majority of lodestar packages and - Functions and variables should be [`camelCase`](https://en.wikipedia.org/wiki/Camel_case), classes should be [`PascalCase`](http://wiki.c2.com/?PascalCase), constants should be `UPPERCASE_WITH_UNDERSCORES`. - Use `"` instead of `'` - All functions should have types declared for all parameters and return value - - You shouldn't be using TypeScript's `any` + - You shouldn't be using TypeScript type `any` - Private class properties should not be prefixed with a `_` - e.g.: `private dirty;`, not `private _dirty;` - Make sure that your code is properly type checked: @@ -145,7 +152,7 @@ We're currently experimenting with hosting the majority of lodestar packages and - Commenting: If your code does something that is not obvious or deviates from standards, leave a comment for other developers to explain your logic and reasoning. - Use `//` commenting format unless it's a comment you want people to see in their IDE. - Use `/** **/` commenting format for documenting a function/variable. -- Code whitespace can be helpful for reading complex code, please add some. +- Code white space can be helpful for reading complex code, please add some. - For unit tests, we forbid import stubbing when other approaches are feasible. ## Tests style guide @@ -187,7 +194,7 @@ Contributors must choose the log level carefully to ensure a consistent experien To edit or extend an existing Grafana dashboard with minimal diff: -1. Grab the .json dashboard file from current unstable +1. Grab the `.json` dashboard file from current unstable 2. Import file to Grafana via the web UI at `/dashboard/import`. Give it some temporal name relevant to your work (i.e. the branch name) 3. Visually edit the dashboard 4. Once done make sure to leave the exact same visual aspect as before: same refresh interval, collapsed rows, etc. @@ -215,7 +222,7 @@ node scripts/download_dashboards.mjs Issues and pull requests are subject to the following labeling guidelines. - PRs may have a status label to indicate deviation from the normal process such as `status-blocked` or `status-do-not-merge` -- Issues and PRs will be tagged with a `scope` and `prio` to indicate type and priority for triaging. +- Issues and PRs will be tagged with a `scope` and `prio` to indicate type and priority for triage. - All other labels allow for further evaluation and organization. Label descriptions can be found below. @@ -224,55 +231,18 @@ Label descriptions can be found below. Status labels apply to issues and pull requests which deviate from normal processes. -- `status-blocked`: This is blocked by another issue that requires resolving first. -- `status-do-not-merge`: Merging this issue will break the build. Do not merge! - ###### `scope.*` Scope Indicator Scope is comparable to Module labels but less strict with the definition of components. It applies to both, issues and pull requests. -- `scope-cpu-performance`: Performance issue and ideas to improve performance. -- `scope-documentation`: All issues related to the Lodestar documentation. -- `scope-interop`: Issues that fix interop issues between Lodestar and CL, EL or tooling. -- `scope-light-clients`: All issues regarding light client development. -- `scope-logging`: Issue about logs: hygiene, format issues, improvements. -- `scope-memory`: Issues to reduce and improve memory usage. -- `scope-metrics`: All issues with regards to the exposed metrics. -- `scope-networking`: All issues related to networking, gossip, and libp2p. -- `scope-profitability`: Issues to directly improve validator performance and its profitability. -- `scope-security`: Issues that fix security issues: DOS, key leak, CVEs. -- `scope-testing`: Issues for adding test coverage, fixing existing tests or testing strategies -- `scope-testnet-debugging`: Issues uncovered through running a node on a public testnet. -- `scope-ux`: Issues for CLI UX or general consumer UX. - ###### `prio.*` Prioritization Indicator A simple indicator of issue prioritization. It mainly applies to issues. -- `prio0-critical`: Drop everything to resolve this immediately. -- `prio1-high`: Resolve issues as soon as possible. -- `prio2-medium`: Resolve this some time soon (tm). -- `prio3-low`: This is nice to have. - ###### `spec.*` Ethereum Consensus Spec Version Target Issues that target a specific version of the Ethereum consensus spec, shall be tagged accordingly. -- `spec-phase0`: Issues targeting the initial Ethereum consensus spec version. -- `spec-altair`: Issues targeting the Altair Ethereum consensus spec version. -- `spec-bellatrix`: Issues targeting the Bellatrix Ethereum consensus spec version. - -###### `meta.*` Meta Labels to organize Miscellaneous Issues - -- `meta-breaking-change`: Introduces breaking changes to DB, Validator, Beacon Node, or CLI interfaces. Handle with care! -- `meta-dependencies`: Pull requests that update a dependency. -- `meta-discussion`: Indicates a topic that requires input from various developers. -- `meta-good-first-issue`: Good first issues for newcomers and first-time contributors. -- `meta-help-wanted`: The author indicates that additional help is wanted. -- `meta-pm`: Issues relating to Project Management tasks. -- `meta-stale`: Label for stale issues applied by the stale bot. -- `meta-technicaldebt`: Issues introducing or resolving technical debts. - ## Community Come chat with us on [Discord](https://discord.gg/aMxzVcr) and join our public weekly planning meetings! diff --git a/README.md b/README.md index 488f078bd4af..6c68e5fda599 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![GitHub release (latest by date)](https://img.shields.io/github/v/release/chainsafe/lodestar?label=Github)](https://github.com/ChainSafe/lodestar/releases/latest) [![npm](https://img.shields.io/npm/v/@chainsafe/lodestar)](https://www.npmjs.com/package/@chainsafe/lodestar) [![Docker Image Version (latest by date)](https://img.shields.io/docker/v/chainsafe/lodestar?color=blue&label=Docker&sort=semver)](https://hub.docker.com/r/chainsafe/lodestar) -[![Eth Consensus Spec v1.1.10](https://img.shields.io/badge/ETH%20consensus--spec-1.1.10-blue)](https://github.com/ethereum/consensus-specs/releases/tag/v1.1.10) +[![Ethereum Consensus Spec v1.1.10](https://img.shields.io/badge/ETH%20consensus--spec-1.1.10-blue)](https://github.com/ethereum/consensus-specs/releases/tag/v1.1.10) [![codecov](https://codecov.io/gh/ChainSafe/lodestar/branch/unstable/graph/badge.svg)](https://codecov.io/gh/ChainSafe/lodestar) ![ES Version](https://img.shields.io/badge/ES-2020-yellow) ![Node Version](https://img.shields.io/badge/node-18.x-green) @@ -45,21 +45,21 @@ yarn build - :package: This mono-repository contains a suite of Ethereum Consensus packages. - :balance_scale: The mono-repository is released under [LGPLv3 license](./LICENSE). Note, that the packages contain their own licenses. -| Package | Version | License | Docs | Description | -| --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | -| [@lodestar/beacon-node](./packages/beacon-node) | [![npm](https://img.shields.io/npm/v/@chainsafe/lodestar)](https://www.npmjs.com/package/@chainsafe/lodestar) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/beacon-node) | :rotating_light: Beacon-chain client | -| [@lodestar/validator](./packages/validator) | [![npm](https://img.shields.io/npm/v/@lodestar/validator)](https://www.npmjs.com/package/@lodestar/validator) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/validator) | :bank: Validator client | -| [@lodestar/light-client](./packages/light-client) | [![npm](https://img.shields.io/npm/v/@lodestar/light-client)](https://www.npmjs.com/package/@lodestar/light-client) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/light-client) | :bird: Ethereum Light client | -| [@lodestar/api](./packages/api) | [![npm](https://img.shields.io/npm/v/@lodestar/api)](https://www.npmjs.com/package/@lodestar/api) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/api) | :clipboard: REST Client for the Eth Beacon API | -| [@chainsafe/lodestar](./packages/cli) | [![npm](https://img.shields.io/npm/v/@chainsafe/lodestar)](https://www.npmjs.com/package/@chainsafe/lodestar) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/cli/) | :computer: Command-line tool for Lodestar | -| [@lodestar/state-transition](./packages/state-transition) | [![npm](https://img.shields.io/npm/v/@lodestar/state-transition)](https://www.npmjs.com/package/@lodestar/state-transition) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/state-transition) | :mag_right: Eth Consensus beacon-state transition | -| [@lodestar/types](./packages/types) | [![npm](https://img.shields.io/npm/v/@lodestar/types)](https://www.npmjs.com/package/@lodestar/types) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/types) | :spiral_notepad: Eth Consensus TypeScript and SSZ types | -| [@lodestar/params](./packages/params) | [![npm](https://img.shields.io/npm/v/@lodestar/params)](https://www.npmjs.com/package/@lodestar/params) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/params) | :spider_web: Eth Consensus network parameters | -| [@lodestar/utils](./packages/utils) | [![npm](https://img.shields.io/npm/v/@lodestar/utils)](https://www.npmjs.com/package/@lodestar/utils) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/utils) | :toolbox: Miscellaneous utility functions used across Lodestar | -| [@lodestar/config](./packages/config) | [![npm](https://img.shields.io/npm/v/@lodestar/config)](https://www.npmjs.com/package/@lodestar/config) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/config) | :spiral_notepad: Eth Consensus types and params bundled together | -| [@lodestar/spec-test-util](./packages/spec-test-util) | [![npm](https://img.shields.io/npm/v/@lodestar/spec-test-util)](https://www.npmjs.com/package/@lodestar/spec-test-util) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/spec-test-util) | :test_tube: Test harness for Eth Consensus spec tests | -| [@lodestar/db](./packages/db) | [![npm](https://img.shields.io/npm/v/@lodestar/db)](https://www.npmjs.com/package/@lodestar/db) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/db) | :floppy_disk: Read/write persistent Eth Consensus data | -| [@lodestar/fork-choice](./packages/fork-choice) | [![npm](https://img.shields.io/npm/v/@lodestar/fork-choice)](https://www.npmjs.com/package/@lodestar/fork-choice) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/fork-choice) | :fork_and_knife: Beacon-chain fork choice | +| Package | Version | License | Docs | Description | +| ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | +| [`@lodestar/beacon-node`](./packages/beacon-node) | [![npm](https://img.shields.io/npm/v/@chainsafe/lodestar)](https://www.npmjs.com/package/@chainsafe/lodestar) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/beacon-node) | :rotating_light: Beacon-chain client | +| [`@lodestar/validator`](./packages/validator) | [![npm](https://img.shields.io/npm/v/@lodestar/validator)](https://www.npmjs.com/package/@lodestar/validator) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/validator) | :bank: Validator client | +| [`@lodestar/light-client`](./packages/light-client) | [![npm](https://img.shields.io/npm/v/@lodestar/light-client)](https://www.npmjs.com/package/@lodestar/light-client) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/light-client) | :bird: Ethereum Light client | +| [`@lodestar/api`](./packages/api) | [![npm](https://img.shields.io/npm/v/@lodestar/api)](https://www.npmjs.com/package/@lodestar/api) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/api) | :clipboard: REST Client for the Ethereum Beacon API | +| [@chainsafe/lodestar`](./packages/cli) | [![npm](https://img.shields.io/npm/v/@chainsafe/lodestar)](https://www.npmjs.com/package/@chainsafe/lodestar) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/cli/) | :computer: Command-line tool for Lodestar | +| [`@lodestar/state-transition`](./packages/state-transition) | [![npm](https://img.shields.io/npm/v/@lodestar/state-transition)](https://www.npmjs.com/package/@lodestar/state-transition) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/state-transition) | :mag_right: Eth Consensus beacon-state transition | +| [`@lodestar/types`](./packages/types) | [![npm](https://img.shields.io/npm/v/@lodestar/types)](https://www.npmjs.com/package/@lodestar/types) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/types) | :spiral_notepad: Eth Consensus TypeScript and SSZ types | +| [`@lodestar/params`](./packages/params) | [![npm](https://img.shields.io/npm/v/@lodestar/params)](https://www.npmjs.com/package/@lodestar/params) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/params) | :spider_web: Eth Consensus network parameters | +| [`@lodestar/utils`](./packages/utils) | [![npm](https://img.shields.io/npm/v/@lodestar/utils)](https://www.npmjs.com/package/@lodestar/utils) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/utils) | :toolbox: Miscellaneous utility functions used across Lodestar | +| [`@lodestar/config`](./packages/config) | [![npm](https://img.shields.io/npm/v/@lodestar/config)](https://www.npmjs.com/package/@lodestar/config) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/config) | :spiral_notepad: Eth Consensus types and params bundled together | +| [`@lodestar/spec-test-util`](./packages/spec-test-util) | [![npm](https://img.shields.io/npm/v/@lodestar/spec-test-util)](https://www.npmjs.com/package/@lodestar/spec-test-util) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/spec-test-util) | :test_tube: Test harness for Eth Consensus spec tests | +| [`@lodestar/db`](./packages/db) | [![npm](https://img.shields.io/npm/v/@lodestar/db)](https://www.npmjs.com/package/@lodestar/db) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/db) | :floppy_disk: Read/write persistent Eth Consensus data | +| [`@lodestar/fork-choice`](./packages/fork-choice) | [![npm](https://img.shields.io/npm/v/@lodestar/fork-choice)](https://www.npmjs.com/package/@lodestar/fork-choice) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/fork-choice) | :fork_and_knife: Beacon-chain fork choice | ## Contributors diff --git a/RELEASE.md b/RELEASE.md index cd88fc160e4a..440b1a13fe82 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -94,6 +94,7 @@ Tagging a stable release will trigger CI to publish to NPM, dockerhub, and Githu - `git checkout stable` - `yarn release:tag-stable 1.1.0` - Must be run locally from a write-access account capable of triggering CI. + #### Manual steps (for example version `v1.1.0`): - Check out the new stable @@ -112,6 +113,7 @@ Tagging a stable release will trigger CI to publish to NPM, dockerhub, and Githu If a stable version requires an immediate hot-fix before the next release, a hot-fix release is started. A similar process for a stable release is used, with the three differences. + - The candidate commit must be chosen from the `stable` branch instead of the `unstable` branch. - Depending on the severity of the bug being fixed, the testing window may be decreased. - All hotfixes are committed with an `unstable` first strategy rather than directly on the RC branch itself. Hotfixes are always merged to `unstable` first, then cherry-picked into hotfix release candidates. @@ -142,7 +144,7 @@ A similar process for a stable release is used, with the three differences. - Commit changes - `git commit -am "v1.1.1"` - `git push origin rc/v1.1.1` -Open draft PR from `rc/v1.1.1` to `stable` with the title `v1.1.1 release`. + Open draft PR from `rc/v1.1.1` to `stable` with the title `v1.1.1 release`. ### 2. Tag release candidate @@ -239,7 +241,9 @@ The release should be announced on the following social channels: - Blog post (if necessary): To outline specific changes that require additional context for users # Release Manager Checklist + This section is to guide the Release Manager tasked with the next version release to ensure all items have been completed. + - Start thread on communication channels for new release - Confirm consensus on `unstable` release candidate commit - Complete Step 1: Create release candidate diff --git a/docs/design/depgraph.md b/docs/design/depgraph.md index ec2e068e383b..9838dfaccac7 100644 --- a/docs/design/depgraph.md +++ b/docs/design/depgraph.md @@ -72,43 +72,43 @@ For a list of all the packages in the monorepo and a description for each, click Let's talk about how each package fits together in finer detail, from top to bottom, following the chart. -## @lodestar/params +## `@lodestar/params` [@lodestar/params](https://github.com/ChainSafe/lodestar/tree/unstable/packages/params) contains the parameters for configuring an Ethereum Consensus network. For example, the [mainnet params](https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/beacon-chain.md#configuration) -## @lodestar/types +## `@lodestar/types` -[@lodestar/types](https://github.com/ChainSafe/lodestar/tree/unstable/packages/types) contains Eth Consensus ssz types and data structures. +[@lodestar/types](https://github.com/ChainSafe/lodestar/tree/unstable/packages/types) contains Ethereum Consensus ssz types and data structures. -## @lodestar/config +## `@lodestar/config` -[@lodestar/config](https://github.com/ChainSafe/lodestar/tree/unstable/packages/config) combines [`lodestar-params`](#chainsafelodestar-params) and [`lodestar-types`](#chainsafelodestar-types) together to be used as a single config object across the other Lodestar packages. +[@lodestar/config](https://github.com/ChainSafe/lodestar/tree/unstable/packages/config) combines `@lodestar/params` and `@lodestar/types` together to be used as a single config object across the other Lodestar packages. -## @lodestar/utils +## `@lodestar/utils` [@lodestar/utils](https://github.com/ChainSafe/lodestar/tree/unstable/packages/utils) contains various utilities that are common among the various Lodestar monorepo packages. -## @lodestar/state-transition +## `@lodestar/state-transition` -[@lodestar/state-transition](https://github.com/ChainSafe/lodestar/tree/unstable/packages/state-transition) contains the Lodestar implementation of the [beacon state transition function](https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function), which is used by [`@lodestar/beacon-node`](#chainsafelodestar) to perform the actual beacon state transition. This package also contains various functions used to calculate info about the beacon chain (such as `computeEpochAtSlot`) which are used by [@lodestar/fork-choice](#chainsafelodestar-fork-choice) and [@lodestar/validator](#chainsafelodestar-validator) +[@lodestar/state-transition](https://github.com/ChainSafe/lodestar/tree/unstable/packages/state-transition) contains the Lodestar implementation of the [beacon state transition function](https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function), which is used by `@lodestar/beacon-node` to perform the actual beacon state transition. This package also contains various functions used to calculate info about the beacon chain (such as `computeEpochAtSlot`) which are used by `@lodestar/fork-choice` and `@lodestar/validator` -## @lodestar/db +## `@lodestar/db` [@lodestar/db](https://github.com/ChainSafe/lodestar/tree/unstable/packages/db) is where all persistent data about the beacon node is stored. Any package that needs to read or write persistent beacon node data depends on `lodestar-db`. -## @lodestar/fork-choice +## `@lodestar/fork-choice` -[@lodestar/fork-choice](https://github.com/ChainSafe/lodestar/tree/unstable/packages/fork-choice) holds the methods for reading/writing the fork choice DAG. The [`@lodestar/beacon-node`](#chainsafelodestar) package is the sole consumer of this package because the beacon node itself is what controls when the fork choice DAG is updated. -For a good explainer on how the fork choice itself works, see the [annotated fork choice spec](https://github.com/ethereum/annotated-spec/blob/v1.1.10/phase0/fork-choice.md). This is an annotated version of the [Eth Consensus fork choice spec](https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/fork-choice.md) which `lodestar-fork-choice` is based on. +[@lodestar/fork-choice](https://github.com/ChainSafe/lodestar/tree/unstable/packages/fork-choice) holds the methods for reading/writing the fork choice DAG. The `@lodestar/beacon-node` package is the sole consumer of this package because the beacon node itself is what controls when the fork choice DAG is updated. +For a good explanation on how the fork choice itself works, see the [annotated fork choice spec](https://github.com/ethereum/annotated-spec/blob/v1.1.10/phase0/fork-choice.md). This is an annotated version of the [Ethereum Consensus fork choice spec](https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/fork-choice.md) which `lodestar-fork-choice` is based on. -## @lodestar/validator +## `@lodestar/validator` -[@lodestar/validator](https://github.com/ChainSafe/lodestar/tree/unstable/packages/validator) contains the validator client. The sole consumer of this package is [@chainsafe/lodestar](#chainsafelodestar-cli), which provides CLI access to run and configure the validator client. However, the validator client communicates to a REST API that is contained in [@lodestar/beacon-node](#chainsafelodestar) (specifically in the `api` module) to perform the validator duties. +[@lodestar/validator](https://github.com/ChainSafe/lodestar/tree/unstable/packages/validator) contains the validator client. The sole consumer of this package is `@chainsafe/lodestar`, which provides CLI access to run and configure the validator client. However, the validator client communicates to a REST API that is contained in `@lodestar/beacon-node` (specifically in the `api` module) to perform the validator duties. -## @lodestar/beacon-node +## `@lodestar/beacon-node` [@lodestar/beacon-node](https://github.com/ChainSafe/lodestar/tree/unstable/packages/beacon-node) contains the actual beacon node process itself, which is the aggregate of all the above packages and the "brain" of the Lodestar beacon chain implementation. All of the node modules live in this package as well. -## @chainsafe/lodestar +## `@chainsafe/lodestar` [@chainsafe/lodestar](https://github.com/ChainSafe/lodestar/tree/unstable/packages/cli) combines everything together for CLI usage and configuration of the beacon node and validator. diff --git a/docs/index.md b/docs/index.md index 2b90a14a7909..82674eb89fe8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,7 +29,7 @@ Hardware specifications minimum / recommended, to run the Lodestar client. ## About these docs -This documentation is open source, contribute at [github.com/chainsafe/lodestar/docs](https://github.com/ChainSafe/lodestar/tree/unstable/docs). +This documentation is open source, contribute at [Github Lodestar repository /docs](https://github.com/ChainSafe/lodestar/tree/unstable/docs). ## Need assistance? diff --git a/docs/install/docker.md b/docs/install/docker.md index 39e8c6aac562..40468e7ad7aa 100644 --- a/docs/install/docker.md +++ b/docs/install/docker.md @@ -1,10 +1,10 @@ # Install with Docker -The [`chainsafe/lodestar`](https://hub.docker.com/r/chainsafe/lodestar) Docker Hub repository is mantained actively. It contains the `lodestar` CLI preinstalled. +The [`chainsafe/lodestar`](https://hub.docker.com/r/chainsafe/lodestar) Docker Hub repository is maintained actively. It contains the `lodestar` CLI preinstalled. !!! info - The Docker Hub image tagged as `chainsafe/lodestar:next` is run on CI every dev commit on our `unstable` branch. + The Docker Hub image tagged as `chainsafe/lodestar:next` is run on CI every commit on our `unstable` branch. For `stable` releases, the image is tagged as `chainsafe/lodestar:latest`. @@ -22,7 +22,8 @@ Pull, run the image and Lodestar should now be ready to use docker pull chainsafe/lodestar docker run chainsafe/lodestar --help ``` + !!! info Docker is the recommended setup for Lodestar. Use our [Lodestar Quickstart scripts](https://github.com/ChainSafe/lodestar-quickstart) with Docker for detailed instructions. - \ No newline at end of file + diff --git a/docs/install/npm.md b/docs/install/npm.md index 8eaa9c51c4af..805141d01523 100644 --- a/docs/install/npm.md +++ b/docs/install/npm.md @@ -3,4 +3,4 @@ !!! danger For mainnet (production) usage, we only recommend installing with docker due to [NPM supply chain attacks](https://hackaday.com/2021/10/22/supply-chain-attack-npm-library-used-by-facebook-and-others-was-compromised/). Until a [safer installation method has been found](https://github.com/ChainSafe/lodestar/issues/3596), do not use this install method except for experimental purposes only. - \ No newline at end of file + diff --git a/docs/install/source.md b/docs/install/source.md index 3ff60ebe7777..6068c5189093 100644 --- a/docs/install/source.md +++ b/docs/install/source.md @@ -10,12 +10,12 @@ Make sure to have [Yarn installed](https://classic.yarnpkg.com/en/docs/install). It is important to make sure the NodeJS version is not changed after reboot by setting a default `nvm alias default && nvm use default`. !!! note - Node Version Manager (NVM) will only install NodeJS for use with the active user. If you intend on setting up Lodestar to run under another user, we recommend using [Nodesource's source for NodeJS](https://github.com/nodesource/distributions/blob/master/README.md#installation-instructions) so you can install NodeJS globally. + Node Version Manager (NVM) will only install NodeJS for use with the active user. If you intend on setting up Lodestar to run under another user, we recommend using [NodeSource's source for NodeJS](https://github.com/nodesource/distributions/blob/master/README.md#installation-instructions) so you can install NodeJS globally. ## Clone repository -Clone the repo locally and build from the stable release branch. +Clone the repository locally and build from the stable release branch. ```bash git clone -b stable https://github.com/chainsafe/lodestar.git diff --git a/docs/libraries/index.md b/docs/libraries/index.md index 530253fd199c..e1576ba542f3 100644 --- a/docs/libraries/index.md +++ b/docs/libraries/index.md @@ -6,29 +6,29 @@ The Lodestar project is divided into Typescript packages that can be used indepe Several useful Ethereum consensus libraries are developed as part of the [Lodestar monorepo](https://github.com/ChainSafe/lodestar) and may be useful when used individually. -- [params](https://github.com/ChainSafe/lodestar/tree/unstable/packages/params) - Ethereum consensus constants and fork names -- [types](https://github.com/ChainSafe/lodestar/tree/unstable/packages/types) - Ethereum consensus datatypes, Typescript interfaces and SSZ type objects -- [config](https://github.com/ChainSafe/lodestar/tree/unstable/packages/config) - Ethereum consensus run-time network configuration -- [api](https://github.com/ChainSafe/lodestar/tree/unstable/packages/api) - Ethereum consensus REST API client -- [flare](https://github.com/ChainSafe/lodestar/tree/unstable/packages/flare) - Beacon chain multi-purpose and debugging tool +- [`params`](https://github.com/ChainSafe/lodestar/tree/unstable/packages/params) - Ethereum consensus constants and fork names +- [`types`](https://github.com/ChainSafe/lodestar/tree/unstable/packages/types) - Ethereum consensus types, Typescript interfaces and SSZ type objects +- [`config`](https://github.com/ChainSafe/lodestar/tree/unstable/packages/config) - Ethereum consensus run-time network configuration +- [`api`](https://github.com/ChainSafe/lodestar/tree/unstable/packages/api) - Ethereum consensus REST API client +- [`flare`](https://github.com/ChainSafe/lodestar/tree/unstable/packages/flare) - Beacon chain multi-purpose and debugging tool ## Other libraries ### BLS Utilities -- [bls](https://github.com/ChainSafe/bls) - Eth Consensus BLS sign / verify / aggregate -- [bls-keystore](https://github.com/ChainSafe/bls-keystore) - store / retrieve a BLS secret key from an [EIP-2335](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2335.md) JSON keystore -- [bls-keygen](https://github.com/ChainSafe/bls-keygen) - utility functions to generate BLS secret keys, following [EIP-2333](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2333.md) and [EIP-2334](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2334.md) -- [bls-hd-key](https://github.com/ChainSafe/bls-hd-key) - low level [EIP-2333](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2333.md) and [EIP-2334](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2334.md) functionality +- [`bls`](https://github.com/ChainSafe/bls) - Ethereum Consensus BLS sign / verify / aggregate +- [`bls-keystore`](https://github.com/ChainSafe/bls-keystore) - store / retrieve a BLS secret key from an [EIP-2335](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2335.md) JSON keystore +- [`bls-keygen`](https://github.com/ChainSafe/bls-keygen) - utility functions to generate BLS secret keys, following [EIP-2333](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2333.md) and [EIP-2334](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2334.md) +- [`bls-hd-key`](https://github.com/ChainSafe/bls-hd-key) - low level [EIP-2333](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2333.md) and [EIP-2334](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2334.md) functionality ### Hashing -- [ssz](https://github.com/ChainSafe/ssz) - Simple Serialize (SSZ) -- [persistent-merkle-tree](https://github.com/ChainSafe/persistent-merkle-tree) - binary merkle tree implemented as a [persistent data structure](https://en.wikipedia.org/wiki/Persistent_data_structure) -- [as-sha256](https://github.com/ChainSafe/as-sha256) - Small AssemblyScript implementation of SHA256 +- [`ssz`](https://github.com/ChainSafe/ssz) - Simple Serialize (SSZ) +- [`persistent-merkle-tree`](https://github.com/ChainSafe/persistent-merkle-tree) - binary merkle tree implemented as a [persistent data structure](https://en.wikipedia.org/wiki/Persistent_data_structure) +- [`as-sha256`](https://github.com/ChainSafe/as-sha256) - Small AssemblyScript implementation of SHA256 ### Networking -- [discv5](https://github.com/ChainSafe/discv5) - [Discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) protocol -- [js-libp2p-gossipsub](https://github.com/ChainSafe/js-libp2p-gossipsub) - [Gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) protocol for js-libp2p -- [js-libp2p-noise](https://github.com/NodeFactoryIo/js-libp2p-noise) - [Noise](https://noiseprotocol.org/noise.html) handshake for js-libp2p +- [`discv5`](https://github.com/ChainSafe/discv5) - [Discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) protocol +- [`js-libp2p-gossipsub`](https://github.com/ChainSafe/js-libp2p-gossipsub) - [Gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) protocol for `js-libp2p` +- [`js-libp2p-noise`](https://github.com/NodeFactoryIo/js-libp2p-noise) - [Noise](https://noiseprotocol.org/noise.html) handshake for `js-libp2p` diff --git a/docs/quickstart.md b/docs/quickstart.md index 02e7b3b2fb5e..57e436452254 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,6 +1,6 @@ ## Lodestar Quickstart -In order to make things easy for users to onboard and try the ethereum **Proof of Stake** we have come up with [Lodestar quickstart](https://github.com/ChainSafe/lodestar-quickstart) scripts! +In order to make things easy for users to onboard and try the Ethereum **Proof of Stake** we have come up with [Lodestar quick start](https://github.com/ChainSafe/lodestar-quickstart) scripts! ✅ Zero Configuration ✅ All testnets supported along with `mainnet` @@ -12,17 +12,11 @@ With just single command you can run lodestar with various execution engines, sw You can adapt them to your production setups with ease! Here is a simple guide for you to follow along: -👉 https://hackmd.io/@philknows/rJegZyH9q +👉 [Lodestar Quick Setup Guide](https://hackmd.io/@philknows/rJegZyH9q) -### Support +### Support We actively maintain and update the configurations of running lodestar with the top of the line execution engines for various PoS networks so you have the minimum possible figuring out to do. -In case you are facing any issues with the quickstart, do reach us out on lodestar discord! +In case you are facing any issues with the quick start guide, do reach us out on lodestar discord! Happy to help! 🙏🙏🙏 - - - - - - diff --git a/docs/tools/flamegraphs.md b/docs/tools/flamegraphs.md index 126ce33b1e64..d6f45303a9aa 100644 --- a/docs/tools/flamegraphs.md +++ b/docs/tools/flamegraphs.md @@ -1,34 +1,30 @@ # Generating Flamegraphs for a Running Node Service on Linux -This guide assumes a running instance of Lodestar and will walk through how to generate a flamegraph for the process while running on Linux. While it is possible to run Lodestar in a number of ways, for performance profiling it is recommended to not use Dockerized implementations. It is best to run Lodestar as a service on a Linux machine. Follow the Lodestar docs to get the service installed and running. Then come back here when you are ready to generate the flamegraph. +This guide assumes a running instance of Lodestar and will walk through how to generate a flamegraph for the process while running on Linux. While it is possible to run Lodestar in a number of ways, for performance profiling it is recommended to not use Dockerized implementations. It is best to run Lodestar as a service on a Linux machine. Follow the Lodestar docs to get the service installed and running. Then come back here when you are ready to generate the flamegraph. ## Modifying Linux and Lodestar -Use the following two commands to install `perf` for generating the stack traces. You may get a warning about needing to restart the VM due to kernel updates. This is nothing to be concerned with and if so, cancel out of the restart dialog. +Use the following two commands to install `perf` for generating the stack traces. You may get a warning about needing to restart the VM due to kernel updates. This is nothing to be concerned with and if so, cancel out of the restart dialog. ```bash sudo apt-get install linux-tools-common linux-tools-generic sudo apt-get install linux-tools-`uname -r` # empirically this throws if run on the same line above ``` -Next we need to update the Lodestar service by modifying the start script. We need to add a necessary flag `--perf-basic-prof` to allow the stack traces to be useful. Node is a virtual machine and `perf` is designed to capture host stack traces. In order to allow the JS functions to be captured meaningfully, `v8` can provide some help. Generally Lodestar is started with a script like the following: +Next we need to update the Lodestar service by modifying the start script. We need to add a necessary flag `--perf-basic-prof` to allow the stack traces to be useful. Node is a virtual machine and `perf` is designed to capture host stack traces. In order to allow the JavaScript functions to be captured meaningfully, `v8` can provide some help. Generally Lodestar is started with a script like the following: ### Example start_lodestar.sh ```sh -#!/bin/bash - -# Add the --perf-basic-prof flag to the node process - node \ - --perf-basic-prof \ + --perf-basic-prof \ --max-old-space-size=4096 \ /usr/src/lodestar/packages/cli/bin/lodestar \ beacon \ --rcConfig /home/devops/beacon/rcconfig.yml ``` -After updating the start script, restart the node process running the beacon service. Note in the command below, that the `beacon` service may have a different name or restart command, depending on your setup. +After updating the start script, restart the node process running the beacon service. Note in the command below, that the `beacon` service may have a different name or restart command, depending on your setup. ```sh admin@12.34.56.78: sudo systemctl restart beacon @@ -64,7 +60,7 @@ The `isolate-*-v8.log` files are the maps that `v8` outputs for the `perf` comma The first command below will run `perf` for 60 seconds, and then save the output to a file named `perf.out`. The second one will merge the exported, unknown, tokens with the isolate maps and output full stack traces for the render. Running both `perf` commands in the folder with the `isolate` maps will allow the data to be seamlessly spliced. Once the output is saved, update the permissions so the file can be copied to your local machine via `scp`. -You can modify the frequency of capture by changing `-F 99` to a different number. Try to stay away from whole numbers as they are more likely to cause interference with periodically scheduled tasks. As an example use `99Hz` or `997Hz` instead of `100Hz` or `1000Hz`. In testing neither seemed to have an appreciable affect on cpu usage when run for a short period of time. +You can modify the frequency of capture by changing `-F 99` to a different number. Try to stay away from whole numbers as they are more likely to cause interference with periodically scheduled tasks. As an example use `99Hz` or `997Hz` instead of `100Hz` or `1000Hz`. In testing neither seemed to have an appreciable affect on CPU usage when run for a short period of time. To change the period of capture adjust the sleep duration (which is in seconds). @@ -76,7 +72,7 @@ admin@12.34.56.78: sudo perf script -f > perf.out admin@12.34.56.78: sudo chmod 777 ~/beacon/perf.out ``` -And then copy the `perf.out` file to your local machine to render the flamegraph. Running at `99Hz` for 180 seconds results in a file size of about 3.5mb and `997Hz` for 60 seconds is roughly 4.4mb. +And then copy the `perf.out` file to your local machine to render the flamegraph. Running at `99Hz` for 180 seconds results in a file size of about 3.5MB and `997Hz` for 60 seconds is roughly 4.4MB. ```sh scp admin@12.34.56.78:/home/devops/beacon/out.perf /some_temp_dir/perf.out @@ -84,15 +80,15 @@ scp admin@12.34.56.78:/home/devops/beacon/out.perf /some_temp_dir/perf.out ## Rendering a Flamegraph -By far the best tool to render flamegraphs is [`flamescope`](https://github.com/Netflix/flamescope) from Netflix. It allows for easy analysis and zooming into specific time periods. It also give a holistic view of how the process is performing over time. +By far the best tool to render flamegraphs is [`flamescope`](https://github.com/Netflix/flamescope) from Netflix. It allows for easy analysis and zooming into specific time periods. It also give a holistic view of how the process is performing over time. ## Installation Python3 is required. Clone the repository and install the dependencies: +_The original is no longer maintained and had a configuration bug. This is a fork that fixes the issue._ + ```sh -# The original is no longer maintained and had a configuration bug -# This is a fork that fixes the issue. git clone https://github.com/matthewkeil/flamescope cd flamescope pip3 install -r requirements.txt @@ -116,7 +112,7 @@ Then navigate in a browser to `http://localhost:8080` and begin analyzing the da ## Filtering Results -There can be a lot of "noise" in the stack traces with `libc`, `v8` and `libuv` calls. It is possible to filter the results to make it more useful, but note this will skew the results. Looking at the graph both filtered and unfiltered can be beneficial. The following `sed` command will remove the noise from the stack traces. You can also use the bottom script programmatically to add/remove filters. +There can be a lot of "noise" in the stack traces with `libc`, `v8` and `libuv` calls. It is possible to filter the results to make it more useful, but note this will skew the results. Looking at the graph both filtered and unfiltered can be beneficial. The following `sed` command will remove the noise from the stack traces. ```sh sed -r -e "/( __libc_start| uv_| LazyCompile | v8::internal::| node::| Builtins_| Builtin:| Stub:| LoadIC:| \\[unknown\\]| LoadPolymorphicIC:)/d" -e 's/ LazyCompile:[*~]?/ /' @@ -137,7 +133,7 @@ sed -r -e "/( __libc_start| uv_| LazyCompile | v8::internal::| node::| Builtins_ - - - -- (this was a great one about filtering methodology) +- (this was a great one about filtering methodology) - ### Visualization Tools diff --git a/docs/usage/beacon-management.md b/docs/usage/beacon-management.md index f0e1f97f8cd4..34709ebcd7f6 100644 --- a/docs/usage/beacon-management.md +++ b/docs/usage/beacon-management.md @@ -106,13 +106,13 @@ If you are starting your node from a blank db/genesis (or from last saved state If you have a synced beacon node available (e.g., your friend's node or an infrastructure provider) and a trusted checkpoint you can rely on, you can start off your beacon node in under a minute! And at the same time kicking the "long range attack" in its butt! -Just supply these **extra args** to your beacon node command: +Just supply these **extra arguments** to your beacon node command: ```bash --checkpointSyncUrl [--wssCheckpoint ] ``` -In case you really trust `checkpointSyncUrl` then you may skip providing `wssCheckpoint`, which will then result into your beacon node syncing and starting off the recently finalized state from the trusted url. +In case you really trust `checkpointSyncUrl` then you may skip providing `wssCheckpoint`, which will then result into your beacon node syncing and starting off the recently finalized state from the trusted URL. !!! warning @@ -131,45 +131,55 @@ necessary for the node to be synced right away to fulfill its duties as there is ### Guide to the sync logs -Lodestar beacon sync log aims to provide information of utmost importance about your node and yet be suucint at the same time. You may see the sync logs in the following format: +Lodestar beacon sync log aims to provide information of utmost importance about your node and yet be succinct at the same time. You may see the sync logs in the following format: `[Sync status] - [ Slot info ] - [Head info] - [Exec block info] - [Finalized info] - [Peers info]` See the following example of different kinds of sync log: + ``` Apr-20 15:24:08.034[] info: Searching peers - peers: 0 - slot: 6265018 - head: 6264018 0xed93…7b0a - exec-block: syncing(17088476 0x9649…) - finalized: 0xbf30…7e7c:195777 Apr-20 15:24:17.000[] info: Searching peers - peers: 0 - slot: 6265019 - head: 6264018 0xed93…7b0a - exec-block: syncing(17088476 0x9649…) - finalized: 0xbf30…7e7c:195777 +``` +``` Apr-20 15:13:41.298[] info: Syncing - 2.5 minutes left - 2.78 slots/s - slot: 6264966 - head: 6262966 0x5cec…f5b8 - exec-block: valid(17088105 0x6f74…) - finalized: 0x5cc0…3874:195764 - peers: 1 Apr-20 15:13:41.298[] info: Syncing - 2 minutes left - 2.78 slots/s - slot: 6264967 - head: 6263965 0x5cec…f5b8 - exec-block: valid(17088105 0x6f74…) - finalized: 0x5cc0…3874:195764 - peers: 1 +``` +``` Apr-20 15:13:53.151[] info: Syncing - 1.6 minutes left - 3.82 slots/s - slot: 6264967 - head: (slot -360) 0xe0cf…9f3c - exec-block: valid(17088167 0x2d6a…) - finalized: 0x8f3f…2f81:195766 - peers: 5 Apr-20 15:14:05.425[] info: Syncing - 1.1 minutes left - 4.33 slots/s - slot: 6264968 - head: (slot -297) 0x3655…1658 - exec-block: valid(17088231 0xdafd…) - finalized: 0x9475…425a:195769 - peers: 2 Apr-20 15:14:53.001[] info: Syncing - 9 seconds left - 5.00 slots/s - slot: 6264972 - head: (slot -45) 0x44e4…20a4 - exec-block: valid(17088475 0xca61…) - finalized: 0x9cbd…ba83:195776 - peers: 8 +``` +``` Apr-20 15:15:01.443[network] info: Subscribed gossip core topics Apr-20 15:15:01.446[sync] info: Subscribed gossip core topics Apr-20 15:15:05.000[] info: Synced - slot: 6264973 - head: 0x90ea…c655 - exec-block: valid(17088521 0xca9b…) - finalized: 0x6981…682f:195778 - peers: 6 Apr-20 15:15:17.003[] info: Synced - slot: 6264974 - head: 0x4f7e…0e3a - exec-block: valid(17088522 0x08b1…) - finalized: 0x6981…682f:195778 - peers: 6 +``` +``` Apr-20 15:15:41.001[] info: Synced - slot: 6264976 - head: (slot -1) 0x17c6…71a7 - exec-block: valid(17088524 0x5bc1…) - finalized: 0x6981…682f:195778 - peers: 8 Apr-20 15:15:53.001[] info: Synced - slot: 6264977 - head: (slot -2) 0x17c6…71a7 - exec-block: valid(17088524 0x5bc1…) - finalized: 0x6981…682f:195778 - peers: 8 +``` +``` Apr-20 15:16:05.000[] info: Synced - slot: 6264978 - head: 0xc9fd…28c5 - exec-block: valid(17088526 0xb5bf…) - finalized: 0x6981…682f:195778 - peers: 8 Apr-20 15:16:17.017[] info: Synced - slot: 6264979 - head: 0xde91…d4cb - exec-block: valid(17088527 0xa488…) - finalized: 0x6981…682f:195778 - peers: 7 - ``` 1. Sync status: Takes three values : `Synced` or `Syncing` (along with sync speed info) or `Searching` if node is is still looking for viable peers from where it can download blocks. 2. Slot (clock) info: What is the current ongoing slot as per the chain genesis -3. Head info: It specifies where the local chain head hash is. In case its far behind the Slot (clock) then it independntly shows the head slot else it show how far behind from the Slot it is if difference < 1000. +3. Head info: It specifies where the local chain head hash is. In case its far behind the Slot (clock) then it independently shows the head slot else it show how far behind from the Slot it is if difference < 1000. -4. Exec block info: It provides the execution information about the head whether its confirmed `valid` or EL is still `syncing` to it, as well as its number and a short hash to easy identification. +4. Execution block info: It provides the execution information about the head whether its confirmed `valid` or execution layer is still `syncing` to it, as well as its number and a short hash to easy identification. 5. Finalized info: What is the current local `finalized` checkpoint in the format of `[checkpoint root]:[checkpoint epoch]`, for e.g.: `0xd7ba…8386:189636` 6. Peer info: Current total number of outbound or inbound peers, for e.g.: `peers: 27` -For more insight into lodestar beacon functioning, you may setup lodestar metrics and use prepared grafana dashboards that you may find in the repo. \ No newline at end of file +For more insight into lodestar beacon functioning, you may setup lodestar metrics and use prepared Grafana dashboards that you may find in the repository. diff --git a/docs/usage/client-monitoring.md b/docs/usage/client-monitoring.md index d319f938b13a..6b35829d6899 100644 --- a/docs/usage/client-monitoring.md +++ b/docs/usage/client-monitoring.md @@ -3,7 +3,7 @@ Lodestar has the ability to send client stats to a remote service for collection. At the moment, the main service offering remote monitoring is [beaconcha.in](https://beaconcha.in/). -Instructions for setting up client monitoring with *beaconcha.in* can be found in their docs about +Instructions for setting up client monitoring with _beaconcha.in_ can be found in their docs about [Mobile App <> Node Monitoring](https://kb.beaconcha.in/beaconcha.in-explorer/mobile-app-less-than-greater-than-beacon-node) and in your [account settings](https://beaconcha.in/user/settings#app). @@ -19,7 +19,7 @@ Client monitoring can be enabled by setting the `--monitoring.endpoint` flag to lodestar beacon --monitoring.endpoint "https://beaconcha.in/api/v1/client/metrics?apikey={apikey}&machine={machineName}" ``` -In case of *beaconcha.in*, the API key can be found in your [account settings](https://beaconcha.in/user/settings#api). +In case of _beaconcha.in_, the API key can be found in your [account settings](https://beaconcha.in/user/settings#api). Setting the machine is optional but it is especially useful if you are monitoring multiple nodes. diff --git a/docs/usage/local.md b/docs/usage/local.md index 85f6d3a28e44..d268e66610c3 100644 --- a/docs/usage/local.md +++ b/docs/usage/local.md @@ -19,15 +19,15 @@ Run a beacon node as a **bootnode**, with 8 validators with the following comman `--genesisValidators` and `--genesisTime` define the genesis state of the beacon chain. `--dataDir` defines a path where lodestar should store the beacon state. -`--enr.ip` sets the enr ip entry for the node (essential for second node to connect via `enr`) and `--enr.udp` exposes the `discv5` discovery service (if you want to connect more than 1 node and enable discovery amongst them via *bootnode*). +`--enr.ip` sets the ENR IP entry for the node (essential for second node to connect via `enr`) and `--enr.udp` exposes the `discv5` discovery service (if you want to connect more than 1 node and enable discovery amongst them via _bootnode_). Lastly the `--reset` flag ensures the state is cleared on each restart - which is useful when testing locally. Once the node has started, make a request to `curl http://localhost:9596/eth/v1/node/identity` and copy the `enr` value. This would be used to connect from the second node. -> ENR stands for ethereum node records, which is a format for conveying p2p connectivity information for ethereum nodes. -> For more info see [eip-778](https://eips.ethereum.org/EIPS/eip-778). +> ENR stands for Ethereum node records, which is a format for conveying p2p connectivity information for Ethereum nodes. +> For more info see [EIP-778](https://eips.ethereum.org/EIPS/eip-778). **Terminal 2** @@ -53,8 +53,8 @@ to have the same beacon chain. Also `--port` and `--rest.port` are supplied since the default values will already be in use by the first node. -The `--network.connectToDiscv5Bootnodes` flags needs to be set to true as this is needed to allow connection to boot enrs on local devnet. -The exact enr of node to connect to is then supplied via the `--network.discv5.bootEnrs` flag. +The `--network.connectToDiscv5Bootnodes` flags needs to be set to true as this is needed to allow connection to boot ENRs on local devnet. +The exact ENR of node to connect to is then supplied via the `--network.discv5.bootEnrs` flag. Once the second node starts, you should see an output similar to the following in either of the terminals: @@ -71,7 +71,6 @@ For example, making the request on the first node via the following command: will give a result similar to the following: ``` - { "data": [ { diff --git a/docs/usage/mev-integration.md b/docs/usage/mev-integration.md index 67f14b8f4c8d..c2f2529edbe6 100644 --- a/docs/usage/mev-integration.md +++ b/docs/usage/mev-integration.md @@ -1,8 +1,8 @@ # MEV & Merge -MEV is a term refered to bundling the transactions in one particular order to extract (mostly) arbitrage opportunities on the DAPPs and DEXes. +MEV is a term that refers to the bundling of transactions in one particular order to extract (mostly) arbitrage opportunities on the dApps and decentralized exchanges. -And the ones who gets to include these execution payloads (miners in pre-merge world, validators in post-merge) in the canonical chain get paid a per-block reward which essentially _should be_ higher than the normal payload inclusion reward (including transactions tips). +And the ones who gets to include these execution payloads (miners before the merge, validators after the merge) in the canonical chain get paid a per-block reward which essentially _should be_ higher than the normal payload inclusion reward (including transactions tips). Currently these happen with miners running forked versions of their favorite execution client, integrating with these "builders" but in the post-merge world they get a more native and standard integration with the CL. @@ -10,7 +10,7 @@ This is what we in CL land refer to as **Builder Api**. ## Lodestar and Builder API -Lodestar offers builder integrations through the _spec-ed_ [builder API](https://ethereum.github.io/builder-specs/#/Builder). +Lodestar offers builder integration through the _spec-ed_ [builder API](https://ethereum.github.io/builder-specs/#/Builder). This sits in parallel with the execution engine so when enabled, lodestar validator run both flows in parallel when its time to propose for a validator key and currently (naively) picks the builder block in preference to execution if a builder block is fetched (else just proceeds with the execution block). @@ -18,14 +18,14 @@ This sits in parallel with the execution engine so when enabled, lodestar valida All you have to do is: -1. Provide lodestar BN with a Builder endpoint (which corresponds to the network you are running) via these additional flags: - ```shell - --builder --builder.urls - ``` -2. Run lodestar VC with these additional flags - ```shell - --builder --suggestedFeeRecipient - ``` +1. Provide lodestar beacon node with a Builder endpoint (which corresponds to the network you are running) via these additional flags: + ```shell + --builder --builder.urls + ``` +2. Run lodestar validator client with these additional flags + ```shell + --builder --suggestedFeeRecipient + ``` There are some more builder flags available in lodestar cli (for both beacon and validator) which you may inspect and use. diff --git a/docs/usage/prometheus-grafana.md b/docs/usage/prometheus-grafana.md index 346fb79f3ebc..681e15d91ede 100644 --- a/docs/usage/prometheus-grafana.md +++ b/docs/usage/prometheus-grafana.md @@ -5,7 +5,7 @@ Prometheus is an open-source monitoring system with efficient time series databa ## Prometheus To start, download Prometheus from https://prometheus.io/download/. -Unzip the downloaded .zip file and run Prometheus from its installed location with the lodestar prometheus.yml passed in as the config file +Unzip the downloaded .zip file and run Prometheus from its installed location with the lodestar `prometheus.yml` passed in as the configuration file ``` ./prometheus --config.file=$dataDir/prometheus.yml @@ -13,7 +13,7 @@ Unzip the downloaded .zip file and run Prometheus from its installed location wi !!! info - 8008 is also the default port specified in the prometheus.yml in the lodestar repo + 8008 is also the default port specified in the `prometheus.yml` in the lodestar repository Then run the Lodestar beacon node with diff --git a/docs/usage/validator-management.md b/docs/usage/validator-management.md index 70479413255d..7bd0b719ee61 100644 --- a/docs/usage/validator-management.md +++ b/docs/usage/validator-management.md @@ -19,7 +19,7 @@ The mnemonic is randomly generated during wallet creation and printed out to the Lodestar is deprecating its functionality to create wallets. -To create a wallet, we recommend using the official [staking-deposit-cli](https://github.com/ethereum/staking-deposit-cli/releases) from the Ethereum Foundation for users comfortable with command line interfaces. +To create a wallet, we recommend using the official [`staking-deposit-cli`](https://github.com/ethereum/staking-deposit-cli/releases) from the Ethereum Foundation for users comfortable with command line interfaces. Alternatively, for a graphical user interface, you can use the [Stakehouse Wagyu Key Generator](https://wagyu.gg/) developed by members of the EthStaker community. @@ -34,20 +34,25 @@ Validators are represented by a BLS keypair. Use your generated mnemonic from on ### Import a validator keystore from your wallet to Lodestar -To import a validator keystore that was created via one of the methods described above, you must locate the validator keystore JSONs exported by those tools (ex. `keystore-m_12381_3600_0_0_0-1654128694.json`). +To import a validator keystore that was created via one of the methods described above, you must locate the validator JSON keystores exported by those tools (ex. `keystore-m_12381_3600_0_0_0-1654128694.json`). -Inside the keystore JSON file, you should have an [EIP-2335 conformant keystore file](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2335.md#json-schema). +Inside the keystore JSON file, you should have an [EIP-2335 keystore file](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2335.md#json-schema). You will also need the passphrase used the encrypt the keystore. This can be specified interactively, or provided in a plaintext file. #### Option 1: Import Keys To Lodestar's Keystores Folder You can load the keys into the keystore folder using the `validator import` command. There are two methods for importing keystores: + +_Interactive passphrase import_ + ```bash -# Interactive passphrase import ./lodestar validator import --importKeystores ./validator_keys +``` + +_Plaintext passphrase file import_ -# Plaintext passphrase file import +```bash ./lodestar validator import --importKeystores ./validator_keys --importKeystoresPassword ./password.txt ``` @@ -61,6 +66,7 @@ You can load the keys into the keystore folder using the `validator import` comm Once imported with either method, these keystores will be automatically loaded when you start the validator. To list the imported keystores, use the `validator list` command. --- + #### Option 2: Import Keys When Starting the Validator To import keys when you start the validator specify the `--importKeystores` and `--importKeystoresPassword` flags with the `validator` command: @@ -74,7 +80,6 @@ To import keys when you start the validator specify the `--importKeystores` and If you import keys using `--importKeystores` at runtime (Option 2) any keys loaded to the keystores folder from Option 1 will be ignored. - ### Configuring the fee recipient address Post-Merge Ethereum requires validators to set a **Fee Recipient** which allows you to receive priority fees when proposing blocks. If you do not set this address, your priority fees will be sent to the [burn address](https://etherscan.io/address/0x0000000000000000000000000000000000000000). @@ -86,6 +91,7 @@ You may choose to use the `--strictFeeRecipientCheck` flag to enable a strict ch ### Submit a validator deposit Please use the official tools to perform your deposits + - `staking-deposit-cli`: - Ethereum Foundation launchpad: diff --git a/package.json b/package.json index 46985e20c8fc..565df68e32b4 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,10 @@ "build:ifchanged": "lerna exec -- ../../scripts/build_if_changed.sh", "lint": "eslint --color --ext .ts packages/*/src packages/*/test", "lint:fix": "yarn lint --fix", + "lint-docs": "prettier '**/*.md' --check", + "lint-docs:fix": "prettier '**/*.md' --write", "check-build": "lerna run check-build", + "check-readme": "lerna run check-readme", "check-types": "lerna run check-types --no-bail", "coverage": "lerna run coverage --no-bail", "test": "lerna run test --no-bail --concurrency 1", @@ -30,8 +33,7 @@ "release:create-rc": "node scripts/release/create_rc.mjs", "release:tag-rc": "node scripts/release/tag_rc.mjs", "release:tag-stable": "node scripts/release/tag_stable.mjs", - "release:publish": "lerna publish from-package --yes --no-verify-access", - "check-readme": "lerna run check-readme" + "release:publish": "lerna publish from-package --yes --no-verify-access" }, "devDependencies": { "@chainsafe/eslint-plugin-node": "^11.2.3", diff --git a/packages/api/README.md b/packages/api/README.md index 02711ffefecf..c8a1ebf3a31a 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -25,7 +25,7 @@ api.beacon "0x933ad9491b62059dd065b560d256d8957a8c402cc6e8d8ee7290ae11e8f7329267a8811c397529dac52ae1342ba58c95" ) .then((res) => { - if(res.ok) { + if (res.ok) { console.log("Your balance is:", res.response.data.balance, res.ok, res.status); } else { console.error(res.status, res.error.code, res.error.message); diff --git a/packages/beacon-node/test/scripts/el-interop/gethdocker/README.md b/packages/beacon-node/test/scripts/el-interop/gethdocker/README.md index da21ad9897f8..df4b56f4368a 100644 --- a/packages/beacon-node/test/scripts/el-interop/gethdocker/README.md +++ b/packages/beacon-node/test/scripts/el-interop/gethdocker/README.md @@ -1,6 +1,7 @@ # Geth Docker setup for running the sim merge tests on local machine ###### Geth docker image + Pull the latest `geth` image from the dockerhub ```bash diff --git a/packages/cli/README.md b/packages/cli/README.md index c780be238f26..5c228733fb8f 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -23,7 +23,7 @@ Here's a quick list of the available CLI commands: | - | - | | `./bin/lodestar init` | Write a configuration and network identity to disk, by default `./.lodestar`| |`./bin/lodestar beacon` | Run a beacon node using a configuration from disk, by default `./.lodestar`| -|`./bin/lodestar account` | Run various subcommands for creating/managing Ethereum Consensus accounts| +|`./bin/lodestar account` | Run various sub-commands for creating/managing Ethereum Consensus accounts| |`./bin/lodestar validator` | Run one or more validator clients| |`./bin/lodestar dev` | Quickly bootstrap a beacon node and multiple validators. Use for development and testing| Append `--help` to any of these commands to print out all options for each command. diff --git a/packages/cli/docsgen/index.ts b/packages/cli/docsgen/index.ts index 005f667ed25d..5e0a3364f73d 100644 --- a/packages/cli/docsgen/index.ts +++ b/packages/cli/docsgen/index.ts @@ -58,7 +58,7 @@ function cmdToMarkdownSection(cmd: CliCommand, parentCommand?: string): Mar body.push("**Options**"); if (cmd.subcommands) { - body.push("The options below apply to all subcommands."); + body.push("The options below apply to all sub-commands."); } // De-duplicate beaconOptions. If all beaconOptions exists in this command, skip them diff --git a/packages/cli/src/cmds/beacon/options.ts b/packages/cli/src/cmds/beacon/options.ts index 27a7762e5b2d..db593e7d7224 100644 --- a/packages/cli/src/cmds/beacon/options.ts +++ b/packages/cli/src/cmds/beacon/options.ts @@ -63,7 +63,7 @@ export const beaconExtraOptions: CliCommandOptions = { wssCheckpoint: { description: - "Start beacon node off a state at the provided weak subjectivity checkpoint, to be supplied in : format. For example, 0x1234:100 will sync and start off from the weakSubjectivity state at checkpoint of epoch 100 with block root 0x1234.", + "Start beacon node off a state at the provided weak subjectivity checkpoint, to be supplied in : format. For example, 0x1234:100 will sync and start off from the weak subjectivity state at checkpoint of epoch 100 with block root 0x1234.", type: "string", group: "weak subjectivity", }, diff --git a/packages/cli/src/cmds/validator/blsToExecutionChange.ts b/packages/cli/src/cmds/validator/blsToExecutionChange.ts index 9f298d10d8a1..3198aad66f65 100644 --- a/packages/cli/src/cmds/validator/blsToExecutionChange.ts +++ b/packages/cli/src/cmds/validator/blsToExecutionChange.ts @@ -37,7 +37,7 @@ like to choose for BLS To Execution Change.", options: { publicKey: { - description: "Validator pubkey for which to set withdrawal address hence enabling withdrawals", + description: "Validator public key for which to set withdrawal address hence enabling withdrawals", type: "string", string: true, }, diff --git a/packages/cli/src/cmds/validator/list.ts b/packages/cli/src/cmds/validator/list.ts index ad789067be43..ae713bcbdecb 100644 --- a/packages/cli/src/cmds/validator/list.ts +++ b/packages/cli/src/cmds/validator/list.ts @@ -15,7 +15,7 @@ export const list: CliCommand = { examples: [ { command: "validator list", - description: "List all validator pubkeys previously imported", + description: "List all validator public keys previously imported", }, ], diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index 117c7d5867c0..609a06164c2c 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -83,31 +83,31 @@ export type KeymanagerArgs = { export const keymanagerOptions: CliCommandOptions = { keymanager: { type: "boolean", - description: "Enable keymanager API server", + description: "Enable key manager API server", default: false, group: "keymanager", }, "keymanager.authEnabled": { type: "boolean", - description: "Enable token bearer authentication for keymanager API server", + description: "Enable token bearer authentication for key manager API server", default: true, group: "keymanager", }, "keymanager.port": { type: "number", - description: "Set port for keymanager API", + description: "Set port for key manager API", defaultDescription: String(keymanagerRestApiServerOptsDefault.port), group: "keymanager", }, "keymanager.address": { type: "string", - description: "Set host for keymanager API", + description: "Set host for key manager API", defaultDescription: keymanagerRestApiServerOptsDefault.address, group: "keymanager", }, "keymanager.cors": { type: "string", - description: "Configures the Access-Control-Allow-Origin CORS header for keymanager API", + description: "Configures the Access-Control-Allow-Origin CORS header for key manager API", defaultDescription: keymanagerRestApiServerOptsDefault.cors, group: "keymanager", }, @@ -207,24 +207,24 @@ export const validatorOptions: CliCommandOptions = { proposerSettingsFile: { description: - "A yaml file to specify detailed default and per validator pubkey customized proposer configs. PS: This feature and its format is in alpha and subject to change", + "A yaml file to specify detailed default and per validator public key customized proposer configs. PS: This feature and its format is in alpha and subject to change", type: "string", }, suggestedFeeRecipient: { description: - "Specify fee recipient default for collecting the EL block fees and rewards (a hex string representing 20 bytes address: ^0x[a-fA-F0-9]{40}$). It would be possible (WIP) to override this per validator key using config or keymanager API. Only used post merge.", + "Specify fee recipient default for collecting the EL block fees and rewards (a hex string representing 20 bytes address: ^0x[a-fA-F0-9]{40}$). It would be possible (WIP) to override this per validator key using config or key manager API. Only used post merge.", defaultDescription: defaultOptions.suggestedFeeRecipient, type: "string", }, strictFeeRecipientCheck: { - description: "Enable strict checking of the validator's feeRecipient with the one returned by engine", + description: "Enable strict checking of the validator's `feeRecipient` with the one returned by engine", type: "boolean", }, defaultGasLimit: { - description: "Suggested gasLimit to the engine/builder for building execution payloads. Only used post merge.", + description: "Suggested gas limit to the engine/builder for building execution payloads. Only used post merge.", defaultDescription: `${defaultOptions.defaultGasLimit}`, type: "number", }, @@ -237,22 +237,22 @@ export const validatorOptions: CliCommandOptions = { "builder.selection": { type: "string", - description: "Default builder block selection strategy: maxprofit, builderalways, or builderonly", - defaultDescription: `${defaultOptions.builderSelection}`, + description: "Default builder block selection strategy: `maxprofit`, `builderalways`, or `builderonly`", + defaultDescription: `\`${defaultOptions.builderSelection}\``, group: "builder", }, importKeystores: { alias: ["keystore"], // Backwards compatibility with old `validator import` cmdx - description: "Path(s) to a directory or single filepath to validator keystores, i.e. Launchpad validators", + description: "Path(s) to a directory or single file path to validator keystores, i.e. Launchpad validators", defaultDescription: "./keystores/*.json", type: "array", }, importKeystoresPassword: { alias: ["passphraseFile"], // Backwards compatibility with old `validator import` cmd - description: "Path to a file with password to decrypt all keystores from importKeystores option", - defaultDescription: "./password.txt", + description: "Path to a file with password to decrypt all keystores from `importKeystores` option", + defaultDescription: "`./password.txt`", type: "string", }, @@ -289,7 +289,7 @@ export const validatorOptions: CliCommandOptions = { "externalSigner.fetch": { conflicts: ["externalSigner.pubkeys"], - description: "Fetch then list of pubkeys to validate from an external signer", + description: "Fetch then list of public keys to validate from an external signer", type: "boolean", group: "externalSignerUrl", }, diff --git a/packages/cli/src/cmds/validator/slashingProtection/export.ts b/packages/cli/src/cmds/validator/slashingProtection/export.ts index e6dc3af49016..0e5b7a17833e 100644 --- a/packages/cli/src/cmds/validator/slashingProtection/export.ts +++ b/packages/cli/src/cmds/validator/slashingProtection/export.ts @@ -37,7 +37,7 @@ export const exportCmd: CliCommand diff --git a/packages/cli/src/cmds/validator/slashingProtection/options.ts b/packages/cli/src/cmds/validator/slashingProtection/options.ts index 4f264ab06632..ff2f109d7d4c 100644 --- a/packages/cli/src/cmds/validator/slashingProtection/options.ts +++ b/packages/cli/src/cmds/validator/slashingProtection/options.ts @@ -9,7 +9,7 @@ export const slashingProtectionOptions: CliCommandOptions diff --git a/packages/cli/src/options/beaconNodeOptions/builder.ts b/packages/cli/src/options/beaconNodeOptions/builder.ts index 7ef671939dcb..7313d836a92c 100644 --- a/packages/cli/src/options/beaconNodeOptions/builder.ts +++ b/packages/cli/src/options/beaconNodeOptions/builder.ts @@ -53,7 +53,7 @@ export const options: CliCommandOptions = { "builder.allowedFaults": { type: "number", - description: "Number of missed slots allowed in the faultInspectionWindow for builder circuit", + description: "Number of missed slots allowed in the `faultInspectionWindow` for builder circuit", group: "builder", }, }; diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index 55e9b8697f1d..dd2049ea00a7 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -51,7 +51,7 @@ export const options: CliCommandOptions = { suggestedFeeRecipient: { type: "string", description: - "Specify fee recipient default for collecting the EL block fees and rewards (a hex string representing 20 bytes address: ^0x[a-fA-F0-9]{40}$) in case validator fails to update for a validator index before calling produceBlock.", + "Specify fee recipient default for collecting the EL block fees and rewards (a hex string representing 20 bytes address: ^0x[a-fA-F0-9]{40}$) in case validator fails to update for a validator index before calling `produceBlock`.", default: defaultOptions.chain.suggestedFeeRecipient, group: "chain", }, @@ -59,7 +59,7 @@ export const options: CliCommandOptions = { emitPayloadAttributes: { type: "boolean", defaultDescription: String(defaultOptions.chain.emitPayloadAttributes), - description: "Flag to SSE emit execution payloadAttributes before every slot", + description: "Flag to SSE emit execution `payloadAttributes` before every slot", group: "chain", }, diff --git a/packages/cli/src/options/beaconNodeOptions/eth1.ts b/packages/cli/src/options/beaconNodeOptions/eth1.ts index aa47c6c3b1a7..c6f6dd9177f8 100644 --- a/packages/cli/src/options/beaconNodeOptions/eth1.ts +++ b/packages/cli/src/options/beaconNodeOptions/eth1.ts @@ -48,7 +48,7 @@ export const options: CliCommandOptions = { "eth1.providerUrls": { description: - "Urls to Eth1 node with enabled rpc. If not explicity provided and execution endpoint provided via execution.urls, it will use execution.urls. Otherwise will try connecting on the specified default(s)", + "Urls to Eth1 node with enabled rpc. If not explicitly provided and execution endpoint provided via execution.urls, it will use execution.urls. Otherwise will try connecting on the specified default(s)", defaultDescription: defaultOptions.eth1.providerUrls?.join(","), type: "array", string: true, diff --git a/packages/cli/src/options/logOptions.ts b/packages/cli/src/options/logOptions.ts index 6eaad79611d9..687057d6ec1e 100644 --- a/packages/cli/src/options/logOptions.ts +++ b/packages/cli/src/options/logOptions.ts @@ -17,7 +17,7 @@ export type LogArgs = { export const logOptions: CliCommandOptions = { logLevel: { choices: LogLevels, - description: "Logging verbosity level for emittings logs to terminal", + description: "Logging verbosity level for emitting logs to terminal", default: LogLevel.info, type: "string", }, @@ -29,7 +29,7 @@ export const logOptions: CliCommandOptions = { logFileLevel: { choices: LogLevels, - description: "Logging verbosity level for emittings logs to file", + description: "Logging verbosity level for emitting logs to file", default: LogLevel.debug, type: "string", }, diff --git a/packages/params/README.md b/packages/params/README.md index 70045f48cb33..ebce8bda5b1b 100644 --- a/packages/params/README.md +++ b/packages/params/README.md @@ -8,7 +8,6 @@ > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project - Lodestar defines all constants and presets defined in the [Ethereum Consensus spec](https://github.com/ethereum/consensus-specs). This can be used in conjunction with other Lodestar libraries to interact with the Ethereum consensus. ## Installation @@ -53,7 +52,7 @@ import {GENESIS_SLOT} from "@lodestar/params"; ### Presets -Presets are "constants"-ish defined in the spec that can only be configured at build-time. These are meant to be treated as constants, and indeed are treated as constants by all downstream Lodestar libraries. The default preset is `mainnet`. The only other preset defined is `minimal`, used only in testing environments. +Presets are defined in the spec as "constantish" and can only be configured at build-time. These are meant to be treated as constants, and indeed are treated as constants by all downstream Lodestar libraries. The default preset is `mainnet`. The only other preset defined is `minimal`, used only in testing environments. The active preset is exported under the `ACTIVE_PRESET` named export. @@ -68,7 +67,7 @@ The preset may be set in one of two ways: Important Notes: -- Interacting with and understanding the active preset is only necessary in very limited testing environments, eg: for ephemeral testnets +- Interacting with and understanding the active preset is only necessary in very limited testing environments, like for ephemeral testnets - The `minimal` preset is NOT compatible with the `mainnet` preset. - using `setActivePreset` may be dangerous, and only should be run once before loading any other libraries. All downstream Lodestar libraries expect the active preset to never change. - Preset values can be overriden by executing `setActivePreset(presetName: PresetName, overrides?: Partial)` and supplying values to override. diff --git a/packages/prover/README.md b/packages/prover/README.md index 9a8f250d8fe3..c4df66751e51 100644 --- a/packages/prover/README.md +++ b/packages/prover/README.md @@ -110,7 +110,7 @@ lodestar-prover start \ ## Warnings -- To use this prover the ehtereum provider must support the `eth_getProof` method. Unfortunately, Infura does not currently support this endpoint. As an alternative, we suggest using Alchemy. +- To use this prover the ethereum provider must support the `eth_getProof` method. Unfortunately, Infura does not currently support this endpoint. As an alternative, we suggest using Alchemy. ## Prerequisites diff --git a/packages/spec-test-util/README.md b/packages/spec-test-util/README.md index 6c1d0aae056a..7b9182a4729c 100644 --- a/packages/spec-test-util/README.md +++ b/packages/spec-test-util/README.md @@ -2,7 +2,7 @@ > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project -Mocha / Chai utility for interacting with eth2.0 spec tests. +Mocha / Chai utility for interacting with eth2.0 spec tests. For usage see [spec tests]("https://github.com/ChainSafe/lodestar/tree/unstable/packages/beacon-node/test/spec") diff --git a/packages/types/README.md b/packages/types/README.md index 1e5c520aa4b7..0c668490bb18 100644 --- a/packages/types/README.md +++ b/packages/types/README.md @@ -8,7 +8,7 @@ > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project -Lodestar defines all datatypes defined in the [Ethereum Consensus spec](https://github.com/ethereum/consensus-specs). This tooling can be used for any Typescript project looking to operate on these types. Both Typescript interfaces _and_ Simple Serialize (SSZ) methods are exported for consumers. +Lodestar defines all data types defined in the [Ethereum Consensus spec](https://github.com/ethereum/consensus-specs). This tooling can be used for any Typescript project looking to operate on these types. Both Typescript interfaces _and_ Simple Serialize (SSZ) methods are exported for consumers. ## Installation @@ -18,14 +18,14 @@ npm install @lodestar/types ## Usage -The lodestar types library organizes datatypes on several dimensions: +The lodestar types library organizes data types on several dimensions: - Typescript interfaces vs SSZ objects - By fork ### Typescript interfaces -Lodestar types are all defined as typescript interfaces. These interfaces can be used independently, and are used throughout downstream Lodestar packages (eg: in the beacon node). +Lodestar types are all defined as typescript interfaces. These interfaces can be used independently, and are used throughout downstream Lodestar packages (in the beacon node). These interfaces are accessible via named exports. @@ -67,7 +67,7 @@ import {Epoch, ssz} from "@lodestar/types"; const epoch: Epoch = ssz.Epoch.defaultValue(); ``` -In some cases, we need interfaces that accept types across all forks, eg: when the fork is not known ahead of time. Typescript interfaces for this purpose are exported under the `allForks` namespace. SSZ Types typed to these interfaces are also provided under an `allForks` namespace, but keyed by `ForkName`. +In some cases, we need interfaces that accept types across all forks, like when the fork is not known ahead of time. Typescript interfaces for this purpose are exported under the `allForks` namespace. SSZ Types typed to these interfaces are also provided under an `allForks` namespace, but keyed by `ForkName`. ```typescript import {ForkName} from "@lodestar/params"; diff --git a/scripts/wordlist_sort.sh b/scripts/wordlist_sort.sh new file mode 100755 index 000000000000..e4713cb402d6 --- /dev/null +++ b/scripts/wordlist_sort.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Define wordlist file +wordlist=".wordlist.txt" + +# Sort the wordlist in place +sort --human-numeric-sort -o "$wordlist" "$wordlist" diff --git a/scripts/wordlist_sort_check.sh b/scripts/wordlist_sort_check.sh new file mode 100755 index 000000000000..578dde398967 --- /dev/null +++ b/scripts/wordlist_sort_check.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# You can sort the wordlist by running: +# ``` +# $ scripts/wordlist_sort.sh +# ``` + +# Define wordlist file +wordlist=".wordlist.txt" + +# Check if wordlist is sorted +if ! sort --ignore-case --human-numeric-sort --check "$wordlist"; then + echo "Error: The wordlist is not sorted." + exit 1 +fi + +# Check for repeated words +if uniq -d "$wordlist" | grep -q .; then + echo "Error: The wordlist contains repeated words." + exit 1 +fi + +echo "The wordlist is sorted and contains no repeated words." \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index db9ee4d9e276..b2da8a922d73 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3757,6 +3757,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.15.11.tgz#b3b790f09cb1696cffcec605de025b088fa4225f" integrity sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q== +"@types/node@^18.16.2": + version "18.16.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.18.tgz#85da09bafb66d4bc14f7c899185336d0c1736390" + integrity sha512-/aNaQZD0+iSBAGnvvN2Cx92HqE5sZCPZtx2TsK+4nvV23fFe09jVDvpArXr2j9DnYlzuU9WuoykDDc6wqvpNcw== + "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" @@ -6264,6 +6269,11 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" +data-uri-to-buffer@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz#d8feb2b2881e6a4f58c2e08acfd0e2834e26222e" + integrity sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A== + datastore-core@^8.0.1: version "8.0.3" resolved "https://registry.yarnpkg.com/datastore-core/-/datastore-core-8.0.3.tgz#d14bda75184b03d319c80d94f49ff36b27a619d4" @@ -7749,6 +7759,14 @@ fecha@^4.2.0: resolved "https://registry.npmjs.org/fecha/-/fecha-4.2.1.tgz" integrity sha512-MMMQ0ludy/nBs1/o0zVOiKTpG7qMbonKUzjJgQFEuvq6INZ1OraKPRAWkBq5vlKLOUMpmNYG1JoN3oDPUQ9m3Q== +fetch-blob@^3.1.2, fetch-blob@^3.1.4: + version "3.2.0" + resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.2.0.tgz#f09b8d4bbd45adc6f0c20b7e787e793e309dcce9" + integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ== + dependencies: + node-domexception "^1.0.0" + web-streams-polyfill "^3.0.3" + figures@3.2.0, figures@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" @@ -7986,6 +8004,13 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +formdata-polyfill@^4.0.10: + version "4.0.10" + resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423" + integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g== + dependencies: + fetch-blob "^3.1.2" + formidable@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/formidable/-/formidable-2.1.2.tgz#fa973a2bec150e4ce7cac15589d7a25fc30ebd89" @@ -11354,6 +11379,11 @@ node-addon-api@^5.0.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-5.1.0.tgz#49da1ca055e109a23d537e9de43c09cca21eb762" integrity sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA== +node-domexception@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" + integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== + node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.npmjs.org/@achingbrain/node-fetch/-/node-fetch-2.6.7.tgz" @@ -11366,6 +11396,15 @@ node-fetch@^2.6.9: dependencies: whatwg-url "^5.0.0" +node-fetch@^3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.3.1.tgz#b3eea7b54b3a48020e46f4f88b9c5a7430d20b2e" + integrity sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow== + dependencies: + data-uri-to-buffer "^4.0.0" + fetch-blob "^3.1.4" + formdata-polyfill "^4.0.10" + node-forge@^1.1.0, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -14984,6 +15023,11 @@ typescript@^5.0.3: resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.0.3.tgz#fe976f0c826a88d0a382007681cbb2da44afdedf" integrity sha512-xv8mOEDnigb/tN9PSMTwSEqAnUvkoXMQlicOb0IUVDBSQCgBSaAAROUZYy2IcUy5qU6XajK5jjjO7TMWqBTKZA== +typescript@^5.0.4: + version "5.1.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.1.3.tgz#8d84219244a6b40b6fb2b33cc1c062f715b9e826" + integrity sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw== + ua-parser-js@^0.7.30: version "0.7.33" resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.33.tgz#1d04acb4ccef9293df6f70f2c3d22f3030d8b532" @@ -15358,6 +15402,11 @@ wcwidth@^1.0.0, wcwidth@^1.0.1: dependencies: defaults "^1.0.3" +web-streams-polyfill@^3.0.3: + version "3.2.1" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz#71c2718c52b45fd49dbeee88634b3a60ceab42a6" + integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q== + web3-bzz@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/web3-bzz/-/web3-bzz-1.10.0.tgz#ac74bc71cdf294c7080a79091079192f05c5baed" From c042a0b8cc6d4fee1a34c69256a4a8d8e1a6017e Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 22 Jun 2023 05:11:41 +0200 Subject: [PATCH 17/96] chore: review fixes for PR-5527 (#5678) * Fix yarn.lock file * Fix readme formatting --- README.md | 2 +- yarn.lock | 49 ------------------------------------------------- 2 files changed, 1 insertion(+), 50 deletions(-) diff --git a/README.md b/README.md index 6c68e5fda599..82af8b433ead 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ yarn build | [`@lodestar/validator`](./packages/validator) | [![npm](https://img.shields.io/npm/v/@lodestar/validator)](https://www.npmjs.com/package/@lodestar/validator) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/validator) | :bank: Validator client | | [`@lodestar/light-client`](./packages/light-client) | [![npm](https://img.shields.io/npm/v/@lodestar/light-client)](https://www.npmjs.com/package/@lodestar/light-client) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/light-client) | :bird: Ethereum Light client | | [`@lodestar/api`](./packages/api) | [![npm](https://img.shields.io/npm/v/@lodestar/api)](https://www.npmjs.com/package/@lodestar/api) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/api) | :clipboard: REST Client for the Ethereum Beacon API | -| [@chainsafe/lodestar`](./packages/cli) | [![npm](https://img.shields.io/npm/v/@chainsafe/lodestar)](https://www.npmjs.com/package/@chainsafe/lodestar) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/cli/) | :computer: Command-line tool for Lodestar | +| [`@chainsafe/lodestar`](./packages/cli) | [![npm](https://img.shields.io/npm/v/@chainsafe/lodestar)](https://www.npmjs.com/package/@chainsafe/lodestar) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/cli/) | :computer: Command-line tool for Lodestar | | [`@lodestar/state-transition`](./packages/state-transition) | [![npm](https://img.shields.io/npm/v/@lodestar/state-transition)](https://www.npmjs.com/package/@lodestar/state-transition) | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/state-transition) | :mag_right: Eth Consensus beacon-state transition | | [`@lodestar/types`](./packages/types) | [![npm](https://img.shields.io/npm/v/@lodestar/types)](https://www.npmjs.com/package/@lodestar/types) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/types) | :spiral_notepad: Eth Consensus TypeScript and SSZ types | | [`@lodestar/params`](./packages/params) | [![npm](https://img.shields.io/npm/v/@lodestar/params)](https://www.npmjs.com/package/@lodestar/params) | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | [![documentation](https://img.shields.io/badge/readme-blue)](./packages/params) | :spider_web: Eth Consensus network parameters | diff --git a/yarn.lock b/yarn.lock index b2da8a922d73..db9ee4d9e276 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3757,11 +3757,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.15.11.tgz#b3b790f09cb1696cffcec605de025b088fa4225f" integrity sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q== -"@types/node@^18.16.2": - version "18.16.18" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.18.tgz#85da09bafb66d4bc14f7c899185336d0c1736390" - integrity sha512-/aNaQZD0+iSBAGnvvN2Cx92HqE5sZCPZtx2TsK+4nvV23fFe09jVDvpArXr2j9DnYlzuU9WuoykDDc6wqvpNcw== - "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" @@ -6269,11 +6264,6 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" -data-uri-to-buffer@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz#d8feb2b2881e6a4f58c2e08acfd0e2834e26222e" - integrity sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A== - datastore-core@^8.0.1: version "8.0.3" resolved "https://registry.yarnpkg.com/datastore-core/-/datastore-core-8.0.3.tgz#d14bda75184b03d319c80d94f49ff36b27a619d4" @@ -7759,14 +7749,6 @@ fecha@^4.2.0: resolved "https://registry.npmjs.org/fecha/-/fecha-4.2.1.tgz" integrity sha512-MMMQ0ludy/nBs1/o0zVOiKTpG7qMbonKUzjJgQFEuvq6INZ1OraKPRAWkBq5vlKLOUMpmNYG1JoN3oDPUQ9m3Q== -fetch-blob@^3.1.2, fetch-blob@^3.1.4: - version "3.2.0" - resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.2.0.tgz#f09b8d4bbd45adc6f0c20b7e787e793e309dcce9" - integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ== - dependencies: - node-domexception "^1.0.0" - web-streams-polyfill "^3.0.3" - figures@3.2.0, figures@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" @@ -8004,13 +7986,6 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" -formdata-polyfill@^4.0.10: - version "4.0.10" - resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423" - integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g== - dependencies: - fetch-blob "^3.1.2" - formidable@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/formidable/-/formidable-2.1.2.tgz#fa973a2bec150e4ce7cac15589d7a25fc30ebd89" @@ -11379,11 +11354,6 @@ node-addon-api@^5.0.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-5.1.0.tgz#49da1ca055e109a23d537e9de43c09cca21eb762" integrity sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA== -node-domexception@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" - integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== - node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.npmjs.org/@achingbrain/node-fetch/-/node-fetch-2.6.7.tgz" @@ -11396,15 +11366,6 @@ node-fetch@^2.6.9: dependencies: whatwg-url "^5.0.0" -node-fetch@^3.3.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.3.1.tgz#b3eea7b54b3a48020e46f4f88b9c5a7430d20b2e" - integrity sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow== - dependencies: - data-uri-to-buffer "^4.0.0" - fetch-blob "^3.1.4" - formdata-polyfill "^4.0.10" - node-forge@^1.1.0, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -15023,11 +14984,6 @@ typescript@^5.0.3: resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.0.3.tgz#fe976f0c826a88d0a382007681cbb2da44afdedf" integrity sha512-xv8mOEDnigb/tN9PSMTwSEqAnUvkoXMQlicOb0IUVDBSQCgBSaAAROUZYy2IcUy5qU6XajK5jjjO7TMWqBTKZA== -typescript@^5.0.4: - version "5.1.3" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.1.3.tgz#8d84219244a6b40b6fb2b33cc1c062f715b9e826" - integrity sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw== - ua-parser-js@^0.7.30: version "0.7.33" resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.33.tgz#1d04acb4ccef9293df6f70f2c3d22f3030d8b532" @@ -15402,11 +15358,6 @@ wcwidth@^1.0.0, wcwidth@^1.0.1: dependencies: defaults "^1.0.3" -web-streams-polyfill@^3.0.3: - version "3.2.1" - resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz#71c2718c52b45fd49dbeee88634b3a60ceab42a6" - integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q== - web3-bzz@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/web3-bzz/-/web3-bzz-1.10.0.tgz#ac74bc71cdf294c7080a79091079192f05c5baed" From 3dd0dfe85200bb1cd682989efc4875a320696238 Mon Sep 17 00:00:00 2001 From: g11tech Date: Thu, 22 Jun 2023 18:14:49 +0530 Subject: [PATCH 18/96] fix: fix max responses that can be received by blobs side car by range (#5681) --- packages/beacon-node/src/network/network.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index e0a2b9103148..3f3adb39076a 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -8,7 +8,7 @@ import {phase0, allForks, deneb, altair, Root, capella, SlotRootHex} from "@lode import {routes} from "@lodestar/api"; import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; import {ResponseIncoming} from "@lodestar/reqresp"; -import {ForkName, ForkSeq} from "@lodestar/params"; +import {ForkName, ForkSeq, MAX_BLOBS_PER_BLOCK} from "@lodestar/params"; import {Metrics, RegistryMetricCreator} from "../metrics/index.js"; import {IBeaconChain} from "../chain/index.js"; import {IBeaconDb} from "../db/interface.js"; @@ -468,7 +468,8 @@ export class Network implements INetwork { ): Promise { return collectMaxResponseTyped( this.sendReqRespRequest(peerId, ReqRespMethod.BlobSidecarsByRange, [Version.V1], request), - request.count, + // request's count represent the slots, so the actual max count received could be slots * blobs per slot + request.count * MAX_BLOBS_PER_BLOCK, responseSszTypeByMethod[ReqRespMethod.BlobSidecarsByRange] ); } From a939aaa59b7bb534009c4f0b75e4eabb90e60fb8 Mon Sep 17 00:00:00 2001 From: g11tech Date: Thu, 22 Jun 2023 18:55:00 +0530 Subject: [PATCH 19/96] fix: update the rate limit quota for blobs req/resp by range/by root methods (#5682) --- .../beacon-node/src/network/reqresp/rateLimit.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/beacon-node/src/network/reqresp/rateLimit.ts b/packages/beacon-node/src/network/reqresp/rateLimit.ts index 6d2ec66777dd..c57876bb7105 100644 --- a/packages/beacon-node/src/network/reqresp/rateLimit.ts +++ b/packages/beacon-node/src/network/reqresp/rateLimit.ts @@ -1,4 +1,9 @@ -import {MAX_REQUEST_BLOCKS, MAX_REQUEST_LIGHT_CLIENT_UPDATES} from "@lodestar/params"; +import { + MAX_REQUEST_BLOCKS, + MAX_REQUEST_LIGHT_CLIENT_UPDATES, + MAX_BLOBS_PER_BLOCK, + MAX_REQUEST_BLOB_SIDECARS, +} from "@lodestar/params"; import {InboundRateLimitQuota} from "@lodestar/reqresp"; import {ReqRespMethod, RequestBodyByMethod} from "./types.js"; import {requestSszTypeByMethod} from "./types.js"; @@ -32,13 +37,13 @@ export const rateLimitQuotas: Record = { getRequestCount: getRequestCountFn(ReqRespMethod.BeaconBlocksByRoot, (req) => req.length), }, [ReqRespMethod.BlobSidecarsByRange]: { - // TODO DENEB: For now same value as BeaconBlocksByRange https://github.com/sigp/lighthouse/blob/bf533c8e42cc73c35730e285c21df8add0195369/beacon_node/lighthouse_network/src/rpc/mod.rs#L118-L130 - byPeer: {quota: MAX_REQUEST_BLOCKS, quotaTimeMs: 10_000}, + // TODO DENEB: For now same value as blobs in BeaconBlocksByRange https://github.com/sigp/lighthouse/blob/bf533c8e42cc73c35730e285c21df8add0195369/beacon_node/lighthouse_network/src/rpc/mod.rs#L118-L130 + byPeer: {quota: MAX_REQUEST_BLOB_SIDECARS, quotaTimeMs: 10_000}, getRequestCount: getRequestCountFn(ReqRespMethod.BlobSidecarsByRange, (req) => req.count), }, [ReqRespMethod.BlobSidecarsByRoot]: { - // TODO DENEB: For now same value as BeaconBlocksByRoot https://github.com/sigp/lighthouse/blob/bf533c8e42cc73c35730e285c21df8add0195369/beacon_node/lighthouse_network/src/rpc/mod.rs#L118-L130 - byPeer: {quota: 128, quotaTimeMs: 10_000}, + // TODO DENEB: For now same value as blobs in BeaconBlocksByRoot https://github.com/sigp/lighthouse/blob/bf533c8e42cc73c35730e285c21df8add0195369/beacon_node/lighthouse_network/src/rpc/mod.rs#L118-L130 + byPeer: {quota: 128 * MAX_BLOBS_PER_BLOCK, quotaTimeMs: 10_000}, getRequestCount: getRequestCountFn(ReqRespMethod.BlobSidecarsByRoot, (req) => req.length), }, [ReqRespMethod.LightClientBootstrap]: { From 7c101f062d24900faa03bc641a5cff2e234974ed Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 22 Jun 2023 09:35:41 -0400 Subject: [PATCH 20/96] chore!: update prometheus-gc-stats dependency (#5677) * chore: update prometheus-gc-stats dependency * chore: use check-latest in setup-node action --- .github/workflows/benchmark.yml | 1 + .github/workflows/docs.yml | 1 + .github/workflows/publish-dev.yml | 1 + .github/workflows/publish-rc.yml | 3 ++- .github/workflows/publish-stable.yml | 1 + .github/workflows/test-browser.yml | 1 + .github/workflows/test-e2e.yml | 1 + .github/workflows/test-sim-merge.yml | 1 + .github/workflows/test-sim.yml | 1 + .github/workflows/test-spec.yml | 1 + .github/workflows/test.yml | 1 + README.md | 2 +- packages/beacon-node/package.json | 3 +-- packages/beacon-node/src/metrics/metrics.ts | 7 ++++-- .../beacon-node/src/metrics/nodeJsMetrics.ts | 8 ++++--- .../src/network/core/networkCoreWorker.ts | 3 ++- .../beacon-node/src/network/discv5/worker.ts | 4 +++- packages/beacon-node/src/node/nodejs.ts | 1 + .../beacon-node/test/unit/metrics/utils.ts | 5 +++- yarn.lock | 24 ++++--------------- 20 files changed, 39 insertions(+), 31 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4cc0b0e9e2a0..d1de888472cf 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -33,6 +33,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 18 + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3dacadfac3ec..98b4ba2b56ee 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,6 +14,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 18 + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index 5fbd0d6439c7..673b2ec6af92 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -22,6 +22,7 @@ jobs: with: node-version: 18 registry-url: "https://registry.npmjs.org" + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index cbb8096e2f82..07106e1945f7 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -52,9 +52,10 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 # Needs full depth for changelog generation - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: node-version: 18 + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 930d65332f05..40f0cba82f78 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -61,6 +61,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 18 + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-browser.yml b/.github/workflows/test-browser.yml index a22091527573..c8854c1896aa 100644 --- a/.github/workflows/test-browser.yml +++ b/.github/workflows/test-browser.yml @@ -29,6 +29,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: ${{matrix.node}} + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 7cddb452e0b6..d76910185489 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -31,6 +31,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: ${{matrix.node}} + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-sim-merge.yml b/.github/workflows/test-sim-merge.yml index 472c11f1c1e1..60cf84560766 100644 --- a/.github/workflows/test-sim-merge.yml +++ b/.github/workflows/test-sim-merge.yml @@ -30,6 +30,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 18 + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-sim.yml b/.github/workflows/test-sim.yml index bfec173c6373..1da1959a8072 100644 --- a/.github/workflows/test-sim.yml +++ b/.github/workflows/test-sim.yml @@ -32,6 +32,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 18 + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 1ff3fbf27bf8..58a80543303f 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -25,6 +25,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 18 + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52d224407de8..6a73ac6841e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,6 +26,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: ${{matrix.node}} + check-latest: true - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/README.md b/README.md index 82af8b433ead..abfcb754e26d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Ethereum Consensus Spec v1.1.10](https://img.shields.io/badge/ETH%20consensus--spec-1.1.10-blue)](https://github.com/ethereum/consensus-specs/releases/tag/v1.1.10) [![codecov](https://codecov.io/gh/ChainSafe/lodestar/branch/unstable/graph/badge.svg)](https://codecov.io/gh/ChainSafe/lodestar) ![ES Version](https://img.shields.io/badge/ES-2020-yellow) -![Node Version](https://img.shields.io/badge/node-18.x-green) +![Node Version](https://img.shields.io/badge/node-18.15.x-green) [![gitpoap badge](https://public-api.gitpoap.io/v1/repo/ChainSafe/lodestar/badge)](https://www.gitpoap.io/gh/ChainSafe/lodestar) [Lodestar](https://lodestar.chainsafe.io) is a TypeScript implementation of the [Ethereum Consensus specification](https://github.com/ethereum/consensus-specs) developed by [ChainSafe Systems](https://chainsafe.io). diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 3ef64b785b1a..39c04a7f9f5f 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -101,6 +101,7 @@ "@chainsafe/libp2p-gossipsub": "^6.2.0", "@chainsafe/libp2p-noise": "^11.0.4", "@chainsafe/persistent-merkle-tree": "^0.5.0", + "@chainsafe/prometheus-gc-stats": "^1.0.0", "@chainsafe/ssz": "^0.10.2", "@chainsafe/threads": "^1.11.0", "@ethersproject/abi": "^5.7.0", @@ -145,7 +146,6 @@ "jwt-simple": "0.5.6", "libp2p": "0.42.2", "prom-client": "^14.2.0", - "prometheus-gc-stats": "^0.6.4", "qs": "^6.11.1", "snappyjs": "^0.7.0", "strict-event-emitter-types": "^2.0.0", @@ -158,7 +158,6 @@ "devDependencies": { "@types/eventsource": "^1.1.11", "@types/leveldown": "^4.0.3", - "@types/prometheus-gc-stats": "^0.6.2", "@types/qs": "^6.9.7", "@types/supertest": "^2.0.12", "@types/tmp": "^0.2.3", diff --git a/packages/beacon-node/src/metrics/metrics.ts b/packages/beacon-node/src/metrics/metrics.ts index 8c38fae72c82..5adb4bffac97 100644 --- a/packages/beacon-node/src/metrics/metrics.ts +++ b/packages/beacon-node/src/metrics/metrics.ts @@ -9,7 +9,9 @@ import {RegistryMetricCreator} from "./utils/registryMetricCreator.js"; import {createValidatorMonitor, ValidatorMonitor} from "./validatorMonitor.js"; import {collectNodeJSMetrics} from "./nodeJsMetrics.js"; -export type Metrics = BeaconMetrics & LodestarMetrics & ValidatorMonitor & {register: RegistryMetricCreator}; +export type Metrics = BeaconMetrics & + LodestarMetrics & + ValidatorMonitor & {register: RegistryMetricCreator; close: () => void}; export function createMetrics( opts: MetricsOptions, @@ -33,7 +35,7 @@ export function createMetrics( lodestar.unhandledPromiseRejections.inc(); }); - collectNodeJSMetrics(register); + const close = collectNodeJSMetrics(register); // Merge external registries for (const externalRegister of externalRegistries) { @@ -47,5 +49,6 @@ export function createMetrics( ...lodestar, ...validatorMonitor, register, + close, }; } diff --git a/packages/beacon-node/src/metrics/nodeJsMetrics.ts b/packages/beacon-node/src/metrics/nodeJsMetrics.ts index 8b06bc87aa30..c565cfc07ba5 100644 --- a/packages/beacon-node/src/metrics/nodeJsMetrics.ts +++ b/packages/beacon-node/src/metrics/nodeJsMetrics.ts @@ -1,7 +1,7 @@ import {collectDefaultMetrics, Registry} from "prom-client"; -import gcStats from "prometheus-gc-stats"; +import {gcStats} from "@chainsafe/prometheus-gc-stats"; -export function collectNodeJSMetrics(register: Registry, prefix?: string): void { +export function collectNodeJSMetrics(register: Registry, prefix?: string): () => void { collectDefaultMetrics({ register, prefix, @@ -13,5 +13,7 @@ export function collectNodeJSMetrics(register: Registry, prefix?: string): void // - nodejs_gc_runs_total: Counts the number of time GC is invoked // - nodejs_gc_pause_seconds_total: Time spent in GC in seconds // - nodejs_gc_reclaimed_bytes_total: The number of bytes GC has freed - gcStats(register, {prefix})(); + // `close` must be called to stop the gc collection process from continuing + const close = gcStats(register, {collectionInterval: 6000, prefix}); + return close; } diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index 565653fbdbfb..38c8ed558b1a 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -50,7 +50,8 @@ const abortController = new AbortController(); // Set up metrics, nodejs and discv5-specific const metricsRegister = workerData.metricsEnabled ? new RegistryMetricCreator() : null; if (metricsRegister) { - collectNodeJSMetrics(metricsRegister, "network_worker_"); + const closeMetrics = collectNodeJSMetrics(metricsRegister, "network_worker_"); + abortController.signal.addEventListener("abort", closeMetrics, {once: true}); } // Main event bus shared across the stack diff --git a/packages/beacon-node/src/network/discv5/worker.ts b/packages/beacon-node/src/network/discv5/worker.ts index ea2d94dce5c3..282128c27614 100644 --- a/packages/beacon-node/src/network/discv5/worker.ts +++ b/packages/beacon-node/src/network/discv5/worker.ts @@ -26,9 +26,10 @@ const logger = getNodeLogger(workerData.loggerOpts); // Set up metrics, nodejs and discv5-specific let metricsRegistry: RegistryMetricCreator | undefined; let enrRelevanceMetric: Gauge<"status"> | undefined; +let closeMetrics: () => void | undefined; if (workerData.metrics) { metricsRegistry = new RegistryMetricCreator(); - collectNodeJSMetrics(metricsRegistry, "discv5_worker_"); + closeMetrics = collectNodeJSMetrics(metricsRegistry, "discv5_worker_"); // add enr relevance metric enrRelevanceMetric = metricsRegistry.gauge<"status">({ @@ -95,6 +96,7 @@ const module: Discv5WorkerApi = { return (await metricsRegistry?.metrics()) ?? ""; }, async close() { + closeMetrics?.(); discv5.removeListener("discovered", onDiscovered); subject.complete(); await discv5.stop(); diff --git a/packages/beacon-node/src/node/nodejs.ts b/packages/beacon-node/src/node/nodejs.ts index 49d4b805c594..43b82a8b769d 100644 --- a/packages/beacon-node/src/node/nodejs.ts +++ b/packages/beacon-node/src/node/nodejs.ts @@ -184,6 +184,7 @@ export class BeaconNode { initBeaconMetrics(metrics, anchorState); // Since the db is instantiated before this, metrics must be injected manually afterwards db.setMetrics(metrics.db); + signal.addEventListener("abort", metrics.close, {once: true}); } const monitoring = opts.monitoring.endpoint diff --git a/packages/beacon-node/test/unit/metrics/utils.ts b/packages/beacon-node/test/unit/metrics/utils.ts index e3f0de3db459..53e61826250c 100644 --- a/packages/beacon-node/test/unit/metrics/utils.ts +++ b/packages/beacon-node/test/unit/metrics/utils.ts @@ -6,5 +6,8 @@ import {testLogger} from "../../utils/logger.js"; export function createMetricsTest(): Metrics { const state = ssz.phase0.BeaconState.defaultViewDU(); const logger = testLogger(); - return createMetrics({enabled: true, port: 0}, config, state, logger); + const metrics = createMetrics({enabled: true, port: 0}, config, state, logger); + // we don't need gc metrics running for tests + metrics.close(); + return metrics; } diff --git a/yarn.lock b/yarn.lock index db9ee4d9e276..19a40cf2e8c0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -627,6 +627,11 @@ resolved "https://registry.npmjs.org/@chainsafe/persistent-ts/-/persistent-ts-0.19.1.tgz" integrity sha512-fUFFFFxdcpYkMAHnjm83EYL/R/smtVmEkJr3FGSI6dwPk4ue9rXjEHf7FTd3V8AbVOcTJGriN4cYf2V+HOYkjQ== +"@chainsafe/prometheus-gc-stats@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@chainsafe/prometheus-gc-stats/-/prometheus-gc-stats-1.0.0.tgz#9404abcf7e7a823596ecf3d71697f644568bde8c" + integrity sha512-l9aKCxQHBBBZIbCAl0y+7D4gded1cHUbIIRips68vN5zgpEuxjojUJSebuhLm+cgYdK1Wvf35gusA802bVfcdQ== + "@chainsafe/ssz@^0.10.2": version "0.10.2" resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.10.2.tgz#c782929e1bb25fec66ba72e75934b31fd087579e" @@ -3774,11 +3779,6 @@ dependencies: "@types/node" "*" -"@types/prometheus-gc-stats@^0.6.2": - version "0.6.2" - resolved "https://registry.yarnpkg.com/@types/prometheus-gc-stats/-/prometheus-gc-stats-0.6.2.tgz#b84246b13d0e7bd8bb61fa97a9ab624306b92697" - integrity sha512-HkT55AB8gPA7mrlkSVEoUgKQmnadWfioPZs0AakXRiggZzGKRCyLHe+WUpnvFashtWOJHUBQDjuR/siT9BKr/Q== - "@types/qs@^6.9.7": version "6.9.7" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" @@ -12044,11 +12044,6 @@ open@^8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -optional@^0.1.3: - version "0.1.4" - resolved "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz" - integrity sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw== - optionator@^0.9.1: version "0.9.1" resolved "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz" @@ -12731,15 +12726,6 @@ prom-client@^14.2.0: dependencies: tdigest "^0.1.1" -prometheus-gc-stats@^0.6.4: - version "0.6.4" - resolved "https://registry.yarnpkg.com/prometheus-gc-stats/-/prometheus-gc-stats-0.6.4.tgz#7326216b92ef71591a535cc31b89ee3f94150fe9" - integrity sha512-HtxtDYRurj7gZS9AqjcfEAldf2e053mh+XW//OjifRxr6Y/aLx8y7ETwWesnJ9DaufvAMyqUUQJUzhB9jLc6vg== - dependencies: - optional "^0.1.3" - optionalDependencies: - gc-stats "^1.4.0" - promise-all-reject-late@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/promise-all-reject-late/-/promise-all-reject-late-1.0.1.tgz#f8ebf13483e5ca91ad809ccc2fcf25f26f8643c2" From 833760876cd87093c1e06617f8c5466b8f36fafb Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 22 Jun 2023 12:19:37 -0400 Subject: [PATCH 21/96] fix: prometheus-gc-stats metrics bugfix (#5683) --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 19a40cf2e8c0..ddbce881cbbe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -628,9 +628,9 @@ integrity sha512-fUFFFFxdcpYkMAHnjm83EYL/R/smtVmEkJr3FGSI6dwPk4ue9rXjEHf7FTd3V8AbVOcTJGriN4cYf2V+HOYkjQ== "@chainsafe/prometheus-gc-stats@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@chainsafe/prometheus-gc-stats/-/prometheus-gc-stats-1.0.0.tgz#9404abcf7e7a823596ecf3d71697f644568bde8c" - integrity sha512-l9aKCxQHBBBZIbCAl0y+7D4gded1cHUbIIRips68vN5zgpEuxjojUJSebuhLm+cgYdK1Wvf35gusA802bVfcdQ== + version "1.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/prometheus-gc-stats/-/prometheus-gc-stats-1.0.2.tgz#585f8f1555251db156d7e50ef8c86dd4f3e78f70" + integrity sha512-h3mFKduSX85XMVbOdWOYvx9jNq99jGcRVNyW5goGOqju1CsI+ZJLhu5z4zBb/G+ksL0R4uLVulu/mIMe7Y0rNg== "@chainsafe/ssz@^0.10.2": version "0.10.2" From 268c9befd00548829dfe5dcef82f7a51711ff7db Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 22 Jun 2023 23:11:34 +0300 Subject: [PATCH 22/96] feat: add block proposal summary metric to validator monitor (#5603) * Add block proposal summary metric to validator monitor * Fix type errors * Fix linter error --------- Co-authored-by: Cayman --- .../src/chain/archiver/archiveBlocks.ts | 32 +-- .../beacon-node/src/chain/archiver/index.ts | 190 +--------------- .../src/chain/beaconProposerCache.ts | 29 +-- packages/beacon-node/src/chain/chain.ts | 2 +- .../beacon-node/src/chain/forkChoice/index.ts | 1 - .../src/metrics/metrics/lodestar.ts | 65 +----- .../src/metrics/validatorMonitor.ts | 87 +++++++- .../opPools/aggregatedAttestationPool.test.ts | 2 - .../collectFinalizedProposalStats.test.ts | 206 ------------------ .../unit/chain/archive/nonCheckpoint.test.ts | 5 +- .../beacon-node/test/utils/mocks/chain.ts | 1 - .../beacon-node/test/utils/typeGenerator.ts | 1 - .../test/utils/validationData/attestation.ts | 1 - .../fork-choice/src/forkChoice/forkChoice.ts | 1 - .../fork-choice/src/protoArray/interface.ts | 3 +- .../fork-choice/test/perf/forkChoice/util.ts | 1 - .../test/unit/forkChoice/forkChoice.test.ts | 2 - .../protoArray/executionStatusUpdates.test.ts | 1 - .../unit/protoArray/getCommonAncestor.test.ts | 2 - .../test/unit/protoArray/protoArray.test.ts | 4 - .../state-transition/src/cache/epochCache.ts | 12 + 21 files changed, 121 insertions(+), 527 deletions(-) delete mode 100644 packages/beacon-node/test/unit/chain/archive/collectFinalizedProposalStats.test.ts diff --git a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts index 76a3f4b91f85..c88aa6cbf5ed 100644 --- a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts @@ -1,6 +1,6 @@ import {fromHexString} from "@chainsafe/ssz"; import {Epoch, Slot, RootHex} from "@lodestar/types"; -import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {IForkChoice} from "@lodestar/fork-choice"; import {Logger, toHex} from "@lodestar/utils"; import {ForkSeq, SLOTS_PER_EPOCH, MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; @@ -18,11 +18,6 @@ const BLOB_SIDECAR_BATCH_SIZE = 32; type BlockRootSlot = {slot: Slot; root: Uint8Array}; type CheckpointHex = {epoch: Epoch; rootHex: RootHex}; -export type FinalizedData = { - finalizedCanonicalCheckpoints: CheckpointHex[]; - finalizedCanonicalBlocks: ProtoBlock[]; - finalizedNonCanonicalBlocks: ProtoBlock[]; -}; /** * Archives finalized blocks from active bucket to archive bucket. @@ -41,7 +36,7 @@ export async function archiveBlocks( logger: Logger, finalizedCheckpoint: CheckpointHex, currentEpoch: Epoch -): Promise { +): Promise { // Use fork choice to determine the blocks to archive and delete // getAllAncestorBlocks response includes the finalized block, so it's also moved to the cold db const finalizedCanonicalBlocks = forkChoice.getAllAncestorBlocks(finalizedCheckpoint.rootHex); @@ -103,8 +98,7 @@ export async function archiveBlocks( } // Prunning potential checkpoint data - const {nonCheckpointBlocks: finalizedCanonicalNonCheckpointBlocks, checkpoints: finalizedCanonicalCheckpoints} = - getNonCheckpointBlocks(finalizedCanonicalBlockRoots); + const finalizedCanonicalNonCheckpointBlocks = getNonCheckpointBlocks(finalizedCanonicalBlockRoots); const nonCheckpointBlockRoots: Uint8Array[] = [...nonCanonicalBlockRoots]; for (const block of finalizedCanonicalNonCheckpointBlocks) { nonCheckpointBlockRoots.push(block.root); @@ -116,15 +110,6 @@ export async function archiveBlocks( totalArchived: finalizedCanonicalBlocks.length, finalizedEpoch: finalizedCheckpoint.epoch, }); - - return { - finalizedCanonicalCheckpoints: finalizedCanonicalCheckpoints.map(({root, epoch}) => ({ - rootHex: toHex(root), - epoch, - })), - finalizedCanonicalBlocks, - finalizedNonCanonicalBlocks, - }; } async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot[]): Promise { @@ -222,9 +207,7 @@ export function getParentRootFromSignedBlock(bytes: Uint8Array): Uint8Array { * @param blocks sequence of linear blocks, from child to ancestor. * In ProtoArray.getAllAncestorNodes child nodes are pushed first to the returned array. */ -export function getNonCheckpointBlocks( - blocks: T[] -): {checkpoints: (T & {epoch: Epoch})[]; nonCheckpointBlocks: T[]} { +export function getNonCheckpointBlocks(blocks: T[]): T[] { // Iterate from lowest child to highest ancestor // Look for the checkpoint of the lowest epoch // If block at `epoch * SLOTS_PER_EPOCH`, it's a checkpoint. @@ -232,11 +215,10 @@ export function getNonCheckpointBlocks( // - Otherwise for the previous epoch the last block is a checkpoint if (blocks.length < 1) { - return {checkpoints: [], nonCheckpointBlocks: []}; + return []; } const nonCheckpointBlocks: T[] = []; - const checkpoints: (T & {epoch: Epoch})[] = []; // Start with Infinity to always trigger `blockEpoch < epochPtr` in the first loop let epochPtr = Infinity; // Assume worst case, since it's unknown if a future epoch will skip the first slot or not. @@ -268,10 +250,8 @@ export function getNonCheckpointBlocks( if (!isCheckpoint) { nonCheckpointBlocks.push(block); - } else { - checkpoints.push({...block, epoch: epochPtrHasFirstSlot ? blockEpoch : blockEpoch + 1}); } } - return {nonCheckpointBlocks, checkpoints}; + return nonCheckpointBlocks; } diff --git a/packages/beacon-node/src/chain/archiver/index.ts b/packages/beacon-node/src/chain/archiver/index.ts index 20ee31da797e..5e9dc314a351 100644 --- a/packages/beacon-node/src/chain/archiver/index.ts +++ b/packages/beacon-node/src/chain/archiver/index.ts @@ -1,16 +1,11 @@ -import {Logger, LogLevel} from "@lodestar/utils"; -import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice"; -import {ValidatorIndex, Slot} from "@lodestar/types"; -import {SLOTS_PER_EPOCH} from "@lodestar/params"; - +import {Logger} from "@lodestar/utils"; +import {CheckpointWithHex} from "@lodestar/fork-choice"; import {IBeaconDb} from "../../db/index.js"; import {JobItemQueue} from "../../util/queue/index.js"; import {IBeaconChain} from "../interface.js"; import {ChainEvent} from "../emitter.js"; -import {Metrics} from "../../metrics/metrics.js"; -import {IStateRegenerator} from "../regen/interface.js"; import {StatesArchiver, StatesArchiverOpts} from "./archiveStates.js"; -import {archiveBlocks, FinalizedData} from "./archiveBlocks.js"; +import {archiveBlocks} from "./archiveBlocks.js"; const PROCESS_FINALIZED_CHECKPOINT_QUEUE_LEN = 256; @@ -48,8 +43,7 @@ export class Archiver { private readonly chain: IBeaconChain, private readonly logger: Logger, signal: AbortSignal, - opts: ArchiverOpts, - private readonly metrics: Metrics | null + opts: ArchiverOpts ) { this.statesArchiver = new StatesArchiver(chain.regen, db, logger, opts); this.prevFinalized = chain.forkChoice.getFinalizedCheckpoint(); @@ -95,7 +89,7 @@ export class Archiver { try { const finalizedEpoch = finalized.epoch; this.logger.verbose("Start processing finalized checkpoint", {epoch: finalizedEpoch, rootHex: finalized.rootHex}); - const finalizedData = await archiveBlocks( + await archiveBlocks( this.chain.config, this.db, this.chain.forkChoice, @@ -104,14 +98,6 @@ export class Archiver { finalized, this.chain.clock.currentEpoch ); - this.collectFinalizedProposalStats( - this.chain.regen, - this.chain.forkChoice, - this.chain.beaconProposerCache, - finalizedData, - finalized, - this.prevFinalized - ); this.prevFinalized = finalized; // should be after ArchiveBlocksTask to handle restart cleanly @@ -174,170 +160,4 @@ export class Archiver { this.logger.error("Error updating backfilledRanges on finalization", {epoch: finalized.epoch}, e as Error); } }; - - private collectFinalizedProposalStats( - regen: IStateRegenerator, - forkChoice: IForkChoice, - beaconProposerCache: IBeaconChain["beaconProposerCache"], - finalizedData: FinalizedData, - finalized: CheckpointWithHex, - lastFinalized: CheckpointWithHex - ): FinalizedStats { - const {finalizedCanonicalCheckpoints, finalizedCanonicalBlocks, finalizedNonCanonicalBlocks} = finalizedData; - - // Range to consider is: - // lastFinalized.epoch * SLOTS_PER_EPOCH + 1, .... finalized.epoch * SLOTS_PER_EPOCH - // So we need to check proposer of lastFinalized (index 1 onwards) as well as 0th index proposer - // of current finalized - const finalizedProposersCheckpoints: FinalizedData["finalizedCanonicalCheckpoints"] = - finalizedCanonicalCheckpoints.filter( - (hexCheck) => hexCheck.epoch < finalized.epoch && hexCheck.epoch > lastFinalized.epoch - ); - finalizedProposersCheckpoints.push(lastFinalized); - finalizedProposersCheckpoints.push(finalized); - - // Sort the data to in following structure to make inferences - const slotProposers = new Map(); - - // 1. Process canonical blocks - for (const block of finalizedCanonicalBlocks) { - // simply set to the single entry as no double proposal can be there for same slot in canonical - slotProposers.set(block.slot, {canonicalVals: [block.proposerIndex], nonCanonicalVals: []}); - } - - // 2. Process non canonical blocks - for (const block of finalizedNonCanonicalBlocks) { - const slotVals = slotProposers.get(block.slot) ?? {canonicalVals: [], nonCanonicalVals: []}; - slotVals.nonCanonicalVals.push(block.proposerIndex); - slotProposers.set(block.slot, slotVals); - } - - // Some simple calculatable stats for all validators - const finalizedCanonicalCheckpointsCount = finalizedCanonicalCheckpoints.length; - const expectedTotalProposalsCount = (finalized.epoch - lastFinalized.epoch) * SLOTS_PER_EPOCH; - const finalizedCanonicalBlocksCount = finalizedCanonicalBlocks.length; - const finalizedOrphanedProposalsCount = finalizedNonCanonicalBlocks.length; - const finalizedMissedProposalsCount = expectedTotalProposalsCount - slotProposers.size; - - const allValidators: ProposalStats = { - total: expectedTotalProposalsCount, - finalized: finalizedCanonicalBlocksCount, - orphaned: finalizedOrphanedProposalsCount, - missed: finalizedMissedProposalsCount, - }; - - // Stats about the attached validators - const attachedProposers = beaconProposerCache - .getProposersSinceEpoch(finalized.epoch) - .map((indexString) => Number(indexString)); - const finalizedAttachedValidatorsCount = attachedProposers.length; - - // Calculate stats for attached validators, based on states in checkpointState cache - let finalizedFoundCheckpointsInStateCache = 0; - - let expectedAttachedValidatorsProposalsCount = 0; - let finalizedAttachedValidatorsProposalsCount = 0; - let finalizedAttachedValidatorsOrphanCount = 0; - let finalizedAttachedValidatorsMissedCount = 0; - - for (const checkpointHex of finalizedProposersCheckpoints) { - const checkpointState = regen.getCheckpointStateSync(checkpointHex); - - // Generate stats for attached validators if we have state info - if (checkpointState !== null) { - finalizedFoundCheckpointsInStateCache++; - - const epochProposers = checkpointState.epochCtx.proposers; - const startSlot = checkpointState.epochCtx.epoch * SLOTS_PER_EPOCH; - - for (let index = 0; index < epochProposers.length; index++) { - const slot = startSlot + index; - - // Let skip processing the slots which are out of range - // Range to consider is: - // lastFinalized.epoch * SLOTS_PER_EPOCH + 1, .... finalized.epoch * SLOTS_PER_EPOCH - if (slot <= lastFinalized.epoch * SLOTS_PER_EPOCH || slot > finalized.epoch * SLOTS_PER_EPOCH) { - continue; - } - - const proposer = epochProposers[index]; - - // If this proposer was attached to this BN for this epoch - if (attachedProposers.includes(proposer)) { - expectedAttachedValidatorsProposalsCount++; - - // Get what validators made canonical/non canonical proposals for this slot - const {canonicalVals, nonCanonicalVals} = slotProposers.get(slot) ?? { - canonicalVals: [], - nonCanonicalVals: [], - }; - let wasFinalized = false; - - if (canonicalVals.includes(proposer)) { - finalizedAttachedValidatorsProposalsCount++; - wasFinalized = true; - } - const attachedProposerNonCanSlotProposals = nonCanonicalVals.filter((nonCanVal) => nonCanVal === proposer); - finalizedAttachedValidatorsOrphanCount += attachedProposerNonCanSlotProposals.length; - - // Check is this slot proposal was missed by this attached validator - if (!wasFinalized && attachedProposerNonCanSlotProposals.length === 0) { - finalizedAttachedValidatorsMissedCount++; - } - } - } - } - } - - const attachedValidators: ProposalStats = { - total: expectedAttachedValidatorsProposalsCount, - finalized: finalizedAttachedValidatorsProposalsCount, - orphaned: finalizedAttachedValidatorsOrphanCount, - missed: finalizedAttachedValidatorsMissedCount, - }; - - this.logger.debug("All validators finalized proposal stats", { - ...allValidators, - finalizedCanonicalCheckpointsCount, - finalizedFoundCheckpointsInStateCache, - }); - - // Only log to info if there is some relevant data to show - // - No need to explicitly track SYNCED state since no validators attached would be there to show - // - debug log if validators attached but no proposals were scheduled - // - info log if proposals were scheduled (canonical) or there were orphans (non canonical) - if (finalizedAttachedValidatorsCount !== 0) { - const logLevel = - attachedValidators.total !== 0 || attachedValidators.orphaned !== 0 ? LogLevel.info : LogLevel.debug; - this.logger[logLevel]("Attached validators finalized proposal stats", { - ...attachedValidators, - validators: finalizedAttachedValidatorsCount, - }); - } else { - this.logger.debug("No proposers attached to beacon node", {finalizedEpoch: finalized.epoch}); - } - - this.metrics?.allValidators.total.set(allValidators.total); - this.metrics?.allValidators.finalized.set(allValidators.finalized); - this.metrics?.allValidators.orphaned.set(allValidators.orphaned); - this.metrics?.allValidators.missed.set(allValidators.missed); - - this.metrics?.attachedValidators.total.set(attachedValidators.total); - this.metrics?.attachedValidators.finalized.set(attachedValidators.finalized); - this.metrics?.attachedValidators.orphaned.set(attachedValidators.orphaned); - this.metrics?.attachedValidators.missed.set(attachedValidators.missed); - - this.metrics?.finalizedCanonicalCheckpointsCount.set(finalizedCanonicalCheckpointsCount); - this.metrics?.finalizedFoundCheckpointsInStateCache.set(finalizedFoundCheckpointsInStateCache); - this.metrics?.finalizedAttachedValidatorsCount.set(finalizedAttachedValidatorsCount); - - // Return stats data for the ease of unit testing - return { - allValidators, - attachedValidators, - finalizedCanonicalCheckpointsCount, - finalizedFoundCheckpointsInStateCache, - finalizedAttachedValidatorsCount, - }; - } } diff --git a/packages/beacon-node/src/chain/beaconProposerCache.ts b/packages/beacon-node/src/chain/beaconProposerCache.ts index 0ed9619c7c5b..59040344f464 100644 --- a/packages/beacon-node/src/chain/beaconProposerCache.ts +++ b/packages/beacon-node/src/chain/beaconProposerCache.ts @@ -8,23 +8,16 @@ const PROPOSER_PRESERVE_EPOCHS = 2; export type ProposerPreparationData = routes.validator.ProposerPreparationData; export class BeaconProposerCache { - private readonly feeRecipientByValidatorIndex: MapDef< - string, - {epoch: Epoch; feeRecipient: string; sinceEpoch: Epoch} - >; + private readonly feeRecipientByValidatorIndex: MapDef; constructor(opts: {suggestedFeeRecipient: string}, private readonly metrics?: Metrics | null) { - this.feeRecipientByValidatorIndex = new MapDef( - () => ({ - epoch: 0, - feeRecipient: opts.suggestedFeeRecipient, - sinceEpoch: 0, - }) - ); + this.feeRecipientByValidatorIndex = new MapDef(() => ({ + epoch: 0, + feeRecipient: opts.suggestedFeeRecipient, + })); } add(epoch: Epoch, {validatorIndex, feeRecipient}: ProposerPreparationData): void { - const sinceEpoch = this.feeRecipientByValidatorIndex.get(validatorIndex)?.sinceEpoch ?? epoch; - this.feeRecipientByValidatorIndex.set(validatorIndex, {epoch, feeRecipient, sinceEpoch}); + this.feeRecipientByValidatorIndex.set(validatorIndex, {epoch, feeRecipient}); } prune(epoch: Epoch): void { @@ -44,14 +37,4 @@ export class BeaconProposerCache { get(proposerIndex: number | string): string | undefined { return this.feeRecipientByValidatorIndex.get(`${proposerIndex}`)?.feeRecipient; } - - getProposersSinceEpoch(epoch: Epoch): ProposerPreparationData["validatorIndex"][] { - const proposers = []; - for (const [validatorIndex, feeRecipientEntry] of this.feeRecipientByValidatorIndex.entries()) { - if (feeRecipientEntry.sinceEpoch <= epoch) { - proposers.push(validatorIndex); - } - } - return proposers; - } } diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 641bd7ac82c6..349b7a95d4f7 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -275,7 +275,7 @@ export class BeaconChain implements IBeaconChain { this.emitter = emitter; this.lightClientServer = lightClientServer; - this.archiver = new Archiver(db, this, logger, signal, opts, metrics); + this.archiver = new Archiver(db, this, logger, signal, opts); // always run PrepareNextSlotScheduler except for fork_choice spec tests if (!opts?.disablePrepareNextSlot) { new PrepareNextSlotScheduler(this, this.config, metrics, this.logger, signal); diff --git a/packages/beacon-node/src/chain/forkChoice/index.ts b/packages/beacon-node/src/chain/forkChoice/index.ts index bc734706413d..4de9d3e6ca23 100644 --- a/packages/beacon-node/src/chain/forkChoice/index.ts +++ b/packages/beacon-node/src/chain/forkChoice/index.ts @@ -65,7 +65,6 @@ export function initializeForkChoice( ProtoArray.initialize( { slot: blockHeader.slot, - proposerIndex: blockHeader.proposerIndex, parentRoot: toHexString(blockHeader.parentRoot), stateRoot: toHexString(blockHeader.stateRoot), blockRoot: toHexString(checkpoint.root), diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index b05e8b6ced6b..fb2d540abb45 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -221,66 +221,6 @@ export function createLodestarMetrics( }), }, - // Finalized block and proposal stats - allValidators: { - total: register.gauge({ - name: "lodestar_all_validators_total_count", - help: "Number of all blocks expected to be finalized", - }), - - orphaned: register.gauge({ - name: "lodestar_all_validators_orphaned_count", - help: "Number of blocks orphaned in the finalization", - }), - - missed: register.gauge({ - name: "lodestar_all_validators_missed_count", - help: "Number of blocks missed in the finalization", - }), - - finalized: register.gauge({ - name: "lodestar_all_validators_finalized_count", - help: "Number of blocks finalized", - }), - }, - - attachedValidators: { - total: register.gauge({ - name: "lodestar_attached_validators_total_count", - help: "Number of blocks expected to be finalized from the attached validators", - }), - - orphaned: register.gauge({ - name: "lodestar_attached_validators_orphaned_count", - help: "Number of blocks orphaned and not finalized from the attached validators", - }), - - missed: register.gauge({ - name: "lodestar_attached_validators_missed_count", - help: "Number of blocks missed in the finalization from the attached validators", - }), - - finalized: register.gauge({ - name: "lodestar_attached_validators_finalized_count", - help: "Number of blocks finalized from the attached validators", - }), - }, - - finalizedCanonicalCheckpointsCount: register.gauge({ - name: "lodestar_finalized_canonical_checkpoints_count", - help: "Number of checkpoints finalized", - }), - - finalizedFoundCheckpointsInStateCache: register.gauge({ - name: "lodestar_finalized_found_checkpoints_in_state_cache", - help: "Number of finalized checkpoints found in state cache including previous finalized", - }), - - finalizedAttachedValidatorsCount: register.gauge({ - name: "lodestar_finalized_attached_validators_count", - help: "Number of proposers attached to the beacon node in the finalization", - }), - // Beacon state transition metrics epochTransitionTime: register.histogram({ @@ -852,6 +792,11 @@ export function createLodestarMetrics( help: "Best guess of the node of the result of previous epoch validators attestation actions and causality", labelNames: ["summary"], }), + prevEpochBlockProposalSummary: register.gauge<"summary">({ + name: "validator_monitor_prev_epoch_block_proposal_summary", + help: "Best guess of the node of the result of previous epoch validators block proposal actions and causality", + labelNames: ["summary"], + }), // Validator Monitor Metrics (real-time) diff --git a/packages/beacon-node/src/metrics/validatorMonitor.ts b/packages/beacon-node/src/metrics/validatorMonitor.ts index 0ef1b361052f..8cd494f47017 100644 --- a/packages/beacon-node/src/metrics/validatorMonitor.ts +++ b/packages/beacon-node/src/metrics/validatorMonitor.ts @@ -25,6 +25,7 @@ const MAX_CACHED_EPOCHS = 4; const MAX_CACHED_DISTINCT_TARGETS = 4; const INTERVALS_LATE_ATTESTATION_SUBMISSION = 1.5; +const INTERVALS_LATE_BLOCK_SUBMISSION = 0.75; const RETAIN_REGISTERED_VALIDATORS_MS = 12 * 3600 * 1000; // 12 hours @@ -157,6 +158,15 @@ type EpochSummary = { syncCommitteeMisses: number; /** Number of times a validator's sync signature was seen in an aggregate */ syncSignatureAggregateInclusions: number; + /** Submitted proposals from this validator at this epoch */ + blockProposals: BlockProposals[]; +}; + +type BlockProposals = { + blockRoot: RootHex; + blockSlot: Slot; + poolSubmitDelaySec: number | null; + successfullyImported: boolean; }; function getEpochSummary(validator: MonitoredValidator, epoch: Epoch): EpochSummary { @@ -176,6 +186,7 @@ function getEpochSummary(validator: MonitoredValidator, epoch: Epoch): EpochSumm syncCommitteeHits: 0, syncCommitteeMisses: 0, syncSignatureAggregateInclusions: 0, + blockProposals: [], }; validator.summaries.set(epoch, summary); } @@ -347,20 +358,42 @@ export function createValidatorMonitor( }, registerBeaconBlock(src, seenTimestampSec, block) { - const index = block.proposerIndex; - const validator = validators.get(index); + const validator = validators.get(block.proposerIndex); // Returns the delay between the start of `block.slot` and `seenTimestamp`. const delaySec = seenTimestampSec - (genesisTime + block.slot * config.SECONDS_PER_SLOT); metrics.gossipBlock.elapsedTimeTillReceived.observe(delaySec); if (validator) { metrics.validatorMonitor.beaconBlockTotal.inc({src}); metrics.validatorMonitor.beaconBlockDelaySeconds.observe({src}, delaySec); + + const summary = getEpochSummary(validator, computeEpochAtSlot(block.slot)); + summary.blockProposals.push({ + blockRoot: toHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)), + blockSlot: block.slot, + poolSubmitDelaySec: delaySec, + successfullyImported: false, + }); } }, registerImportedBlock(block, {proposerBalanceDelta}) { - if (validators.has(block.proposerIndex)) { + const validator = validators.get(block.proposerIndex); + if (validator) { metrics.validatorMonitor.proposerBalanceDeltaKnown.observe(proposerBalanceDelta); + + // There should be alredy a summary for the block. Could be missing when using one VC multiple BNs + const summary = getEpochSummary(validator, computeEpochAtSlot(block.slot)); + const proposal = summary.blockProposals.find((p) => p.blockSlot === block.slot); + if (proposal) { + proposal.successfullyImported = true; + } else { + summary.blockProposals.push({ + blockRoot: toHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)), + blockSlot: block.slot, + poolSubmitDelaySec: null, + successfullyImported: true, + }); + } } }, @@ -575,12 +608,12 @@ export function createValidatorMonitor( // Compute summaries of previous epoch attestation performance const prevEpoch = Math.max(0, computeEpochAtSlot(headState.slot) - 1); + const rootCache = new RootHexCache(headState); if (config.getForkSeq(headState.slot) >= ForkSeq.altair) { const {previousEpochParticipation} = headState as CachedBeaconStateAltair; const prevEpochStartSlot = computeStartSlotAtEpoch(prevEpoch); const prevEpochTargetRoot = toHex(getBlockRootAtSlot(headState, prevEpochStartSlot)); - const rootCache = new RootHexCache(headState); // Check attestation performance for (const [index, validator] of validators.entries()) { @@ -591,6 +624,20 @@ export function createValidatorMonitor( }); } } + + if (headState.epochCtx.proposersPrevEpoch !== null) { + // proposersPrevEpoch is null on the first epoch of `headState` being generated + for (const [slotIndex, validatorIndex] of headState.epochCtx.proposersPrevEpoch.entries()) { + const validator = validators.get(validatorIndex); + if (validator) { + // If expected proposer is a tracked validator + const summary = validator.summaries.get(prevEpoch); + metrics.validatorMonitor.prevEpochAttestationSummary.inc({ + summary: renderBlockProposalSummary(config, rootCache, summary, SLOTS_PER_EPOCH * prevEpoch + slotIndex), + }); + } + } + } }, /** @@ -901,6 +948,38 @@ function isMissedSlot(rootCache: RootHexCache, slot: Slot): boolean { return slot > 0 && rootCache.getBlockRootAtSlot(slot) === rootCache.getBlockRootAtSlot(slot - 1); } +function renderBlockProposalSummary( + config: ChainConfig, + rootCache: RootHexCache, + summary: EpochSummary | undefined, + proposalSlot: Slot +): string { + const proposal = summary?.blockProposals.find((proposal) => proposal.blockSlot === proposalSlot); + if (!proposal) { + return "not_submitted"; + } + + if (rootCache.getBlockRootAtSlot(proposalSlot) === proposal.blockRoot) { + // Cannonical state includes our block + return "cannonical"; + } + + let out = "orphaned"; + + if (isMissedSlot(rootCache, proposalSlot)) { + out += "_missed"; + } + + if ( + proposal.poolSubmitDelaySec !== null && + proposal.poolSubmitDelaySec > (INTERVALS_LATE_BLOCK_SUBMISSION * config.SECONDS_PER_SLOT) / INTERVALS_PER_SLOT + ) { + out += "_late"; + } + + return out; +} + /** * Cache to prevent accessing the state tree to fetch block roots repeteadly. * In normal network conditions the same root is read multiple times, specially the target. diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 0441d0806b6d..77c7bcd9948c 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -60,7 +60,6 @@ describe("getAttestationsForBlock", () => { protoArray = ProtoArray.initialize( { slot: blockHeader.slot, - proposerIndex: 0, parentRoot: toHexString(blockHeader.parentRoot), stateRoot: toHexString(blockHeader.stateRoot), blockRoot: toHexString(checkpoint.root), @@ -84,7 +83,6 @@ describe("getAttestationsForBlock", () => { protoArray.onBlock( { slot, - proposerIndex: 0, blockRoot: toHexString(getBlockRootAtSlot(originalState, slot)), parentRoot: toHexString(getBlockRootAtSlot(originalState, slot - 1)), stateRoot: toHexString(originalState.stateRoots.get(slot % HISTORICAL_ROOTS_LIMIT)), diff --git a/packages/beacon-node/test/unit/chain/archive/collectFinalizedProposalStats.test.ts b/packages/beacon-node/test/unit/chain/archive/collectFinalizedProposalStats.test.ts deleted file mode 100644 index 8039efd047fe..000000000000 --- a/packages/beacon-node/test/unit/chain/archive/collectFinalizedProposalStats.test.ts +++ /dev/null @@ -1,206 +0,0 @@ -import {expect} from "chai"; -import sinon from "sinon"; -import {Slot, Epoch, ValidatorIndex} from "@lodestar/types"; -import {ForkChoice, ProtoBlock, CheckpointWithHex} from "@lodestar/fork-choice"; -import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; -import {ZERO_HASH_HEX, ZERO_HASH} from "../../../../src/constants/index.js"; -import {StubbedBeaconDb, StubbedChainMutable} from "../../../utils/stub/index.js"; -import {testLogger} from "../../../utils/logger.js"; -import {Archiver, FinalizedStats} from "../../../../src/chain/archiver/index.js"; -import {FinalizedData} from "../../../../src/chain/archiver/archiveBlocks.js"; -import {BeaconChain, CheckpointHex} from "../../../../src/chain/index.js"; -import {BeaconProposerCache} from "../../../../src/chain/beaconProposerCache.js"; -import {generateCachedState} from "../../../utils/state.js"; -import {QueuedStateRegenerator} from "../../../../src/chain/regen/queued.js"; - -describe("collectFinalizedProposalStats", function () { - const logger = testLogger(); - - let chainStub: StubbedChainMutable<"forkChoice" | "emitter" | "beaconProposerCache" | "regen">; - let dbStub: StubbedBeaconDb; - let cpStateCache: Map; - // let beaconProposerCacheStub = SinonStubbedInstance & BeaconProposerCache; - let archiver: Archiver; - - beforeEach(function () { - cpStateCache = new Map(); - const regen = sinon.createStubInstance(QueuedStateRegenerator); - regen.getCheckpointStateSync.callsFake((cp) => cpStateCache.get(cpKey(cp)) ?? null); - chainStub = sinon.createStubInstance(BeaconChain) as typeof chainStub; - chainStub.forkChoice = sinon.createStubInstance(ForkChoice); - const suggestedFeeRecipient = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - chainStub.beaconProposerCache = new BeaconProposerCache({suggestedFeeRecipient}); - chainStub.regen = regen; - const controller = new AbortController(); - - dbStub = new StubbedBeaconDb(); - archiver = new Archiver( - dbStub as Archiver["db"], - chainStub as Archiver["chain"], - logger, - controller.signal, - {archiveStateEpochFrequency: 10, disableArchiveOnCheckpoint: true}, - null - ); - }); - - // Each test case is: - // name, canonical slots [], non canonical slots [], finalized checkpoints [], - // [prev finalized, latest finalized], attached validators[], exected stats - const testCases1: [string, Slot[], Slot[], Epoch[], [Epoch, Epoch], ValidatorIndex[], FinalizedStats][] = [ - [ - "allVals - E:16, F:15, O:1, M:0 attachVals(8) - E:16, F:15, O:1, M:0", - [16, 15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], - [14], - [2, 1], - [0, 2], - [1, 2, 3, 4, 5, 6, 7, 8], - { - allValidators: {total: 16, finalized: 15, orphaned: 1, missed: 0}, - attachedValidators: {total: 16, finalized: 15, orphaned: 1, missed: 0}, - finalizedCanonicalCheckpointsCount: 2, - finalizedFoundCheckpointsInStateCache: 3, - finalizedAttachedValidatorsCount: 8, - }, - ], - [ - "allVals - E:16, F:15, O:0, M:1 attachVals(8) - E:16, F:15, O:0, M:1", - [16, 15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], - [], - [2, 1], - [0, 2], - [1, 2, 3, 4, 5, 6, 7, 8], - { - allValidators: {total: 16, finalized: 15, orphaned: 0, missed: 1}, - attachedValidators: {total: 16, finalized: 15, orphaned: 0, missed: 1}, - finalizedCanonicalCheckpointsCount: 2, - finalizedFoundCheckpointsInStateCache: 3, - finalizedAttachedValidatorsCount: 8, - }, - ], - [ - "allVals - E:8, F:6, O:1, M:1 attachVals(8) - E:8, F:6, O:1, M:1", - [16, 14, 12, 11, 10, 9], - [15], - [2], - [1, 2], - [1, 2, 3, 4, 5, 6, 7, 8], - { - allValidators: {total: 8, finalized: 6, orphaned: 1, missed: 1}, - attachedValidators: {total: 8, finalized: 6, orphaned: 1, missed: 1}, - finalizedCanonicalCheckpointsCount: 1, - finalizedFoundCheckpointsInStateCache: 2, - finalizedAttachedValidatorsCount: 8, - }, - ], - [ - "allVals - E:16, F:16, O:0, M:0 attachVals(8) - E:16, F:16, O:0, M:0", - [24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9], - [], - [3, 2], - [1, 3], - [1, 2, 3, 4, 5, 6, 7, 8], - { - allValidators: {total: 16, finalized: 16, orphaned: 0, missed: 0}, - attachedValidators: {total: 16, finalized: 16, orphaned: 0, missed: 0}, - finalizedCanonicalCheckpointsCount: 2, - finalizedFoundCheckpointsInStateCache: 3, - finalizedAttachedValidatorsCount: 8, - }, - ], - [ - "allVals - E:8, F:7, O:1, M:0 attachVals(6) - E:6, F:6, O:0, M:0", - [7, 6, 5, 4, 3, 2, 1], - [8], - [1], - [0, 1], - [3, 4, 5, 6, 7, 8], - { - allValidators: {total: 8, finalized: 7, orphaned: 1, missed: 0}, - attachedValidators: {total: 6, finalized: 6, orphaned: 0, missed: 0}, - finalizedCanonicalCheckpointsCount: 1, - finalizedFoundCheckpointsInStateCache: 2, - finalizedAttachedValidatorsCount: 6, - }, - ], - [ - "allVals - E:8, F:7, O:0, M:1 attachVals(6) - E:6, F:6, O:0, M:0", - [7, 6, 5, 4, 3, 2, 1], - [], - [1], - [0, 1], - [3, 4, 5, 6, 7, 8], - { - allValidators: {total: 8, finalized: 7, orphaned: 0, missed: 1}, - attachedValidators: {total: 6, finalized: 6, orphaned: 0, missed: 0}, - finalizedCanonicalCheckpointsCount: 1, - finalizedFoundCheckpointsInStateCache: 2, - finalizedAttachedValidatorsCount: 6, - }, - ], - ]; - const allValidators = [1, 2, 3, 4, 5, 6, 7, 8]; - - for (const [ - id, - finalizedCanonicalBlockSlots, - finalizedNonCanonicalBlockSlots, - finalizedCanonicalCheckpointEpochs, - [prevFinalizedEpoch, finalizedEpoch], - attachedValidators, - expectedStats, - ] of testCases1) { - it(id, async function () { - const finalizedCanonicalBlocks = finalizedCanonicalBlockSlots.map(makeBlock); - const finalizedNonCanonicalBlocks = finalizedNonCanonicalBlockSlots.map(makeBlock); - const finalizedCanonicalCheckpoints = finalizedCanonicalCheckpointEpochs.map(makeCheckpoint); - const prevFinalized = makeCheckpoint(prevFinalizedEpoch); - const finalized = makeCheckpoint(finalizedEpoch); - - addtoBeaconCache(chainStub["beaconProposerCache"], finalized.epoch, attachedValidators); - addDummyStateCache(prevFinalized, allValidators); - finalizedCanonicalCheckpoints.forEach((eachCheckpoint) => { - addDummyStateCache(eachCheckpoint, allValidators); - }); - - const finalizedData = {finalizedCanonicalCheckpoints, finalizedCanonicalBlocks, finalizedNonCanonicalBlocks}; - const processedStats = archiver["collectFinalizedProposalStats"]( - chainStub.regen, - chainStub.forkChoice, - chainStub.beaconProposerCache, - finalizedData as FinalizedData, - finalized, - prevFinalized - ); - - expect(expectedStats).to.deep.equal(processedStats); - }); - } - - function addDummyStateCache(checkpoint: CheckpointHex, proposers: number[]): void { - const checkpointstate = generateCachedState(); - checkpointstate.epochCtx.proposers = proposers; - checkpointstate.epochCtx.epoch = checkpoint.epoch; - cpStateCache.set(cpKey(checkpoint), checkpointstate); - } - - function cpKey(cp: CheckpointHex): string { - return JSON.stringify(cp); - } -}); - -function makeBlock(slot: Slot): ProtoBlock { - return {slot: slot, proposerIndex: (slot % 8) + 1} as ProtoBlock; -} - -function makeCheckpoint(epoch: Epoch): CheckpointWithHex { - return {epoch, rootHex: ZERO_HASH_HEX, root: ZERO_HASH}; -} - -function addtoBeaconCache(cache: BeaconProposerCache, epoch: number, proposers: ValidatorIndex[]): void { - const suggestedFeeRecipient = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - - proposers.forEach((eachProposer) => { - cache.add(epoch, {validatorIndex: `${eachProposer}`, feeRecipient: suggestedFeeRecipient}); - }); -} diff --git a/packages/beacon-node/test/unit/chain/archive/nonCheckpoint.test.ts b/packages/beacon-node/test/unit/chain/archive/nonCheckpoint.test.ts index e9c1f01bc756..6b26d83d9e59 100644 --- a/packages/beacon-node/test/unit/chain/archive/nonCheckpoint.test.ts +++ b/packages/beacon-node/test/unit/chain/archive/nonCheckpoint.test.ts @@ -34,10 +34,9 @@ describe("chain / archive / getNonCheckpointBlocks", () => { // blocks are to be passed in reverse order as thats how they would be recieved in // ProtoArray.getAllAncestorNodes - const {nonCheckpointBlocks, checkpoints} = getNonCheckpointBlocks(blocks.reverse().map(toProtoBlock)); + const nonAncestorBlocks = getNonCheckpointBlocks(blocks.reverse().map(toProtoBlock)); - expect(sort(nonCheckpointBlocks.map((block) => block.slot))).to.deep.equal(sort(nonCheckpointSlots)); - expect(sort(checkpoints.map((block) => block.slot))).to.deep.equal(sort(maybeCheckpointSlots)); + expect(sort(nonAncestorBlocks.map((block) => block.slot))).to.deep.equal(sort(nonCheckpointSlots)); }); } }); diff --git a/packages/beacon-node/test/utils/mocks/chain.ts b/packages/beacon-node/test/utils/mocks/chain.ts index 523b300f2735..9704b3a3f825 100644 --- a/packages/beacon-node/test/utils/mocks/chain.ts +++ b/packages/beacon-node/test/utils/mocks/chain.ts @@ -15,7 +15,6 @@ export function getMockForkChoice(): StubbedOf = {}): ProtoBl ...overrides, slot: 0, - proposerIndex: 0, blockRoot: ZERO_HASH_HEX, parentRoot: ZERO_HASH_HEX, stateRoot: ZERO_HASH_HEX, diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index c18acd54ed23..f27d3c9c020c 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -53,7 +53,6 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { // Add block to forkChoice const headBlock: ProtoBlock = { slot: attSlot, - proposerIndex: 0, blockRoot: toHexString(beaconBlockRoot), parentRoot: ZERO_HASH_HEX, stateRoot: ZERO_HASH_HEX, diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index e7edb4dcf68b..cba427b22c57 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -445,7 +445,6 @@ export class ForkChoice implements IForkChoice { // it can still be identified as the head even if it doesn't have any votes. const protoBlock: ProtoBlock = { slot: slot, - proposerIndex: block.proposerIndex, blockRoot: blockRootHex, parentRoot: parentRootHex, targetRoot: toHexString(targetRoot), diff --git a/packages/fork-choice/src/protoArray/interface.ts b/packages/fork-choice/src/protoArray/interface.ts index b822cb631631..87a37a7b6ff7 100644 --- a/packages/fork-choice/src/protoArray/interface.ts +++ b/packages/fork-choice/src/protoArray/interface.ts @@ -1,4 +1,4 @@ -import {Epoch, Slot, RootHex, UintNum64, ValidatorIndex} from "@lodestar/types"; +import {Epoch, Slot, RootHex, UintNum64} from "@lodestar/types"; // RootHex is a root as a hex string // Used for lightweight and easy comparison @@ -53,7 +53,6 @@ export type ProtoBlock = BlockExecution & { * This is useful for upstream fork choice logic. */ slot: Slot; - proposerIndex: ValidatorIndex; blockRoot: RootHex; parentRoot: RootHex; /** diff --git a/packages/fork-choice/test/perf/forkChoice/util.ts b/packages/fork-choice/test/perf/forkChoice/util.ts index 22d2aa5e5ef8..c8d3472ec3f4 100644 --- a/packages/fork-choice/test/perf/forkChoice/util.ts +++ b/packages/fork-choice/test/perf/forkChoice/util.ts @@ -57,7 +57,6 @@ export function initializeForkChoice(opts: Opts): ForkChoice { const blockRoot = "0x" + String(slot).padStart(64, "0"); const block: ProtoBlock = { slot: genesisSlot + slot, - proposerIndex: 0, blockRoot, parentRoot: parentBlockRoot, stateRoot: blockRoot, diff --git a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts index 42e6189d443b..966d3a27556c 100644 --- a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts @@ -20,7 +20,6 @@ const rootBlockBytePrefix = 0xbb; describe("Forkchoice", function () { const genesisSlot = 0; const genesisEpoch = 0; - const proposerIndex = 0; const genesisRoot = "0x0000000000000000000000000000000000000000000000000000000000000000"; const finalizedRoot = getBlockRoot(genesisSlot); @@ -85,7 +84,6 @@ describe("Forkchoice", function () { const getBlock = (slot: number, skippedSlots: number[] = []): ProtoBlock => { return { slot, - proposerIndex: proposerIndex, blockRoot: getBlockRoot(slot), parentRoot: getParentBlockRoot(slot, skippedSlots), stateRoot: getStateRoot(slot), diff --git a/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts b/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts index 1b16e17023c8..e1dda450aa46 100644 --- a/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts +++ b/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts @@ -91,7 +91,6 @@ function setupForkChoice(): ProtoArray { fc.onBlock( { slot: block.slot, - proposerIndex: 0, blockRoot: block.root, parentRoot: block.parent, stateRoot: "-", diff --git a/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts b/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts index 90340acba79c..3d47d906f74a 100644 --- a/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts +++ b/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts @@ -27,7 +27,6 @@ describe("getCommonAncestor", () => { const fc = ProtoArray.initialize( { slot: 0, - proposerIndex: 0, stateRoot: "-", parentRoot: "-", blockRoot: "0", @@ -50,7 +49,6 @@ describe("getCommonAncestor", () => { fc.onBlock( { slot: block.slot, - proposerIndex: 0, blockRoot: block.root, parentRoot: block.parent, stateRoot: "-", diff --git a/packages/fork-choice/test/unit/protoArray/protoArray.test.ts b/packages/fork-choice/test/unit/protoArray/protoArray.test.ts index 51e1168cfffc..88d6453e6204 100644 --- a/packages/fork-choice/test/unit/protoArray/protoArray.test.ts +++ b/packages/fork-choice/test/unit/protoArray/protoArray.test.ts @@ -8,7 +8,6 @@ describe("ProtoArray", () => { const genesisSlot = 0; const genesisEpoch = 0; - const proposerIndex = 0; const stateRoot = "0"; const finalizedRoot = "1"; const parentRoot = "1"; @@ -18,7 +17,6 @@ describe("ProtoArray", () => { const fc = ProtoArray.initialize( { slot: genesisSlot, - proposerIndex: proposerIndex, stateRoot, parentRoot, blockRoot: finalizedRoot, @@ -41,7 +39,6 @@ describe("ProtoArray", () => { fc.onBlock( { slot: genesisSlot + 1, - proposerIndex: proposerIndex, blockRoot: finalizedDesc, parentRoot: finalizedRoot, stateRoot, @@ -65,7 +62,6 @@ describe("ProtoArray", () => { fc.onBlock( { slot: genesisSlot + 1, - proposerIndex: proposerIndex, blockRoot: notFinalizedDesc, parentRoot: unknown, stateRoot, diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index ef738d69f6a6..da41631afe29 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -104,6 +104,9 @@ export class EpochCache { */ proposers: ValidatorIndex[]; + /** Proposers for previous epoch, initialized to null in first epoch */ + proposersPrevEpoch: ValidatorIndex[] | null; + /** * The next proposer seed is only used in the getBeaconProposersNextEpoch call. It cannot be moved into * getBeaconProposersNextEpoch because it needs state as input and all data needed by getBeaconProposersNextEpoch @@ -190,6 +193,7 @@ export class EpochCache { pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; proposers: number[]; + proposersPrevEpoch: number[] | null; proposersNextEpoch: ProposersDeferred; previousShuffling: EpochShuffling; currentShuffling: EpochShuffling; @@ -213,6 +217,7 @@ export class EpochCache { this.pubkey2index = data.pubkey2index; this.index2pubkey = data.index2pubkey; this.proposers = data.proposers; + this.proposersPrevEpoch = data.proposersPrevEpoch; this.proposersNextEpoch = data.proposersNextEpoch; this.previousShuffling = data.previousShuffling; this.currentShuffling = data.currentShuffling; @@ -388,6 +393,8 @@ export class EpochCache { pubkey2index, index2pubkey, proposers, + // On first epoch, set to null to prevent unnecessary work since this is only used for metrics + proposersPrevEpoch: null, proposersNextEpoch, previousShuffling, currentShuffling, @@ -423,6 +430,7 @@ export class EpochCache { index2pubkey: this.index2pubkey, // Immutable data proposers: this.proposers, + proposersPrevEpoch: this.proposersPrevEpoch, proposersNextEpoch: this.proposersNextEpoch, previousShuffling: this.previousShuffling, currentShuffling: this.currentShuffling, @@ -468,6 +476,10 @@ export class EpochCache { epochTransitionCache.nextEpochShufflingActiveValidatorIndices, nextEpoch ); + + // Roll current proposers into previous proposers for metrics + this.proposersPrevEpoch = this.proposers; + const currentProposerSeed = getSeed(state, this.currentShuffling.epoch, DOMAIN_BEACON_PROPOSER); this.proposers = computeProposers(currentProposerSeed, this.currentShuffling, this.effectiveBalanceIncrements); From 7483c39008f1d01e4d54dd92eac4e20108159962 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 23 Jun 2023 14:32:47 +0200 Subject: [PATCH 23/96] fix(validator): close gc collection process on graceful shutdown (#5686) --- packages/cli/src/cmds/validator/handler.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index a9f60a422bd5..69ead593d1e6 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -117,7 +117,8 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr // Collect NodeJS metrics defined in the Lodestar repo if (metrics) { - collectNodeJSMetrics(register); + const closeMetrics = collectNodeJSMetrics(register); + onGracefulShutdownCbs.push(() => closeMetrics()); // only start server if metrics are explicitly enabled if (args["metrics"]) { From 85ac6cb99fa8ed832720b0c18d7dbbf98df0ca10 Mon Sep 17 00:00:00 2001 From: Cayman Date: Fri, 23 Jun 2023 14:34:19 -0400 Subject: [PATCH 24/96] chore!: set node version requirement to >=18.15.0 (#5690) chore!: set node requirement to >=18.15.x --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 565df68e32b4..4e7497462b35 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "root", "private": true, "engines": { - "node": ">=12.9.0" + "node": ">=18.15.0" }, "workspaces": [ "packages/*" From 50daa01c80b22136e70b7cfb5ab66c735f9641a8 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 23 Jun 2023 20:58:39 +0200 Subject: [PATCH 25/96] chore: use availableParallelism to determine thread pool size (#5691) --- packages/beacon-node/src/chain/bls/multithread/poolSize.ts | 2 +- .../src/cmds/validator/keymanager/decryptKeystores/poolSize.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/chain/bls/multithread/poolSize.ts b/packages/beacon-node/src/chain/bls/multithread/poolSize.ts index e32c976c6687..9397f97849cd 100644 --- a/packages/beacon-node/src/chain/bls/multithread/poolSize.ts +++ b/packages/beacon-node/src/chain/bls/multithread/poolSize.ts @@ -4,7 +4,7 @@ try { if (typeof navigator !== "undefined") { defaultPoolSize = navigator.hardwareConcurrency ?? 4; } else { - defaultPoolSize = (await import("node:os")).cpus().length; + defaultPoolSize = (await import("node:os")).availableParallelism(); } } catch (e) { defaultPoolSize = 8; diff --git a/packages/cli/src/cmds/validator/keymanager/decryptKeystores/poolSize.ts b/packages/cli/src/cmds/validator/keymanager/decryptKeystores/poolSize.ts index 4a3475a725cb..9219ae6d8537 100644 --- a/packages/cli/src/cmds/validator/keymanager/decryptKeystores/poolSize.ts +++ b/packages/cli/src/cmds/validator/keymanager/decryptKeystores/poolSize.ts @@ -4,8 +4,7 @@ try { if (typeof navigator !== "undefined") { maxPoolSize = navigator.hardwareConcurrency ?? 4; } else { - // TODO change this line to use os.availableParallelism() once we upgrade to node v20 - maxPoolSize = (await import("node:os")).cpus().length; + maxPoolSize = (await import("node:os")).availableParallelism(); } } catch (e) { maxPoolSize = 8; From f5d12a893f5b913a04938275bb248ace1ca47539 Mon Sep 17 00:00:00 2001 From: g11tech Date: Mon, 26 Jun 2023 12:32:33 +0530 Subject: [PATCH 26/96] feat: implement eip 4788 for deneb (#5703) * feat: implement eip 4788 for deneb * fixes * lint --- .../blocks/verifyBlocksExecutionPayloads.ts | 8 +- .../beacon-node/src/chain/prepareNextSlot.ts | 1 + .../chain/produceBlock/produceBlockBody.ts | 77 +++++++++++++------ .../beacon-node/src/execution/engine/http.ts | 19 +++-- .../src/execution/engine/interface.ts | 6 +- .../beacon-node/src/execution/engine/types.ts | 8 +- packages/types/src/allForks/sszTypes.ts | 2 +- packages/types/src/allForks/types.ts | 9 ++- packages/types/src/bellatrix/sszTypes.ts | 3 +- packages/types/src/deneb/sszTypes.ts | 18 +++++ packages/types/src/deneb/types.ts | 1 + 11 files changed, 114 insertions(+), 38 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts index 1f6fc125a44a..ccfc917421b2 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts @@ -293,7 +293,13 @@ export async function verifyBlockExecutionPayload( ForkSeq[fork] >= ForkSeq.deneb ? (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments.map(kzgCommitmentToVersionedHash) : undefined; - const execResult = await chain.executionEngine.notifyNewPayload(fork, executionPayloadEnabled, versionedHashes); + const parentBlockRoot = ForkSeq[fork] >= ForkSeq.deneb ? block.message.parentRoot : undefined; + const execResult = await chain.executionEngine.notifyNewPayload( + fork, + executionPayloadEnabled, + versionedHashes, + parentBlockRoot + ); chain.metrics?.engineNotifyNewPayloadResult.inc({result: execResult.status}); diff --git a/packages/beacon-node/src/chain/prepareNextSlot.ts b/packages/beacon-node/src/chain/prepareNextSlot.ts index e15d159f43ba..38a97efd38a2 100644 --- a/packages/beacon-node/src/chain/prepareNextSlot.ts +++ b/packages/beacon-node/src/chain/prepareNextSlot.ts @@ -147,6 +147,7 @@ export class PrepareNextSlotScheduler { this.chain, this.logger, fork as ForkExecution, // State is of execution type + fromHex(headRoot), safeBlockHash, finalizedBlockHash, prepareState, diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 521c80bdaa23..e8a76ffc64b1 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -28,7 +28,7 @@ import { } from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {ForkSeq, ForkExecution, isForkExecution} from "@lodestar/params"; -import {toHex, sleep, Logger, fromHex} from "@lodestar/utils"; +import {toHex, sleep, Logger} from "@lodestar/utils"; import type {BeaconChain} from "../chain.js"; import {PayloadId, IExecutionEngine, IExecutionBuilder, PayloadAttributes} from "../../execution/index.js"; @@ -154,6 +154,7 @@ export async function produceBlockBody( this, this.logger, fork, + parentBlockRoot, safeBlockHash, finalizedBlockHash ?? ZERO_HASH_HEX, currentState as CachedBeaconStateBellatrix, @@ -196,6 +197,7 @@ export async function produceBlockBody( this, this.logger, fork, + parentBlockRoot, safeBlockHash, finalizedBlockHash ?? ZERO_HASH_HEX, currentState as CachedBeaconStateExecutions, @@ -316,6 +318,7 @@ export async function prepareExecutionPayload( }, logger: Logger, fork: ForkExecution, + parentBlockRoot: Root, safeBlockHash: RootHex, finalizedBlockHash: RootHex, state: CachedBeaconStateExecutions, @@ -357,15 +360,12 @@ export async function prepareExecutionPayload( prepType = PayloadPreparationType.Fresh; } - const attributes: PayloadAttributes = { - timestamp, - prevRandao, - suggestedFeeRecipient, - }; - - if (ForkSeq[fork] >= ForkSeq.capella) { - attributes.withdrawals = getExpectedWithdrawals(state as CachedBeaconStateCapella).withdrawals; - } + const attributes: PayloadAttributes = preparePayloadAttributes(fork, chain, { + prepareState: state, + prepareSlot: state.slot, + parentBlockRoot, + feeRecipient: suggestedFeeRecipient, + }); payloadId = await chain.executionEngine.notifyForkchoiceUpdate( fork, @@ -469,19 +469,12 @@ export async function getPayloadAttributesForSSE( if (!parentHashRes.isPremerge) { const {parentHash} = parentHashRes; - const timestamp = computeTimeAtSlot(chain.config, prepareSlot, prepareState.genesisTime); - const prevRandao = getRandaoMix(prepareState, prepareState.epochCtx.epoch); - const payloadAttributes = { - timestamp, - prevRandao, - suggestedFeeRecipient: fromHex(feeRecipient), - }; - - if (ForkSeq[fork] >= ForkSeq.capella) { - (payloadAttributes as capella.SSEPayloadAttributes["payloadAttributes"]).withdrawals = getExpectedWithdrawals( - prepareState as CachedBeaconStateCapella - ).withdrawals; - } + const payloadAttributes = preparePayloadAttributes(fork, chain, { + prepareState, + prepareSlot, + parentBlockRoot, + feeRecipient, + }); const ssePayloadAttributes: allForks.SSEPayloadAttributes = { proposerIndex: prepareState.epochCtx.getBeaconProposer(prepareSlot), @@ -497,4 +490,42 @@ export async function getPayloadAttributesForSSE( } } +function preparePayloadAttributes( + fork: ForkExecution, + chain: { + config: ChainForkConfig; + }, + { + prepareState, + prepareSlot, + parentBlockRoot, + feeRecipient, + }: { + prepareState: CachedBeaconStateExecutions; + prepareSlot: Slot; + parentBlockRoot: Root; + feeRecipient: string; + } +): allForks.SSEPayloadAttributes["payloadAttributes"] { + const timestamp = computeTimeAtSlot(chain.config, prepareSlot, prepareState.genesisTime); + const prevRandao = getRandaoMix(prepareState, prepareState.epochCtx.epoch); + const payloadAttributes = { + timestamp, + prevRandao, + suggestedFeeRecipient: feeRecipient, + }; + + if (ForkSeq[fork] >= ForkSeq.capella) { + (payloadAttributes as capella.SSEPayloadAttributes["payloadAttributes"]).withdrawals = getExpectedWithdrawals( + prepareState as CachedBeaconStateCapella + ).withdrawals; + } + + if (ForkSeq[fork] >= ForkSeq.deneb) { + (payloadAttributes as deneb.SSEPayloadAttributes["payloadAttributes"]).parentBeaconBlockRoot = parentBlockRoot; + } + + return payloadAttributes; +} + /** process_sync_committee_contributions is implemented in syncCommitteeContribution.getSyncAggregate */ diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index 11ab396e6cd4..35c54a96418e 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -1,4 +1,4 @@ -import {RootHex, allForks, Wei} from "@lodestar/types"; +import {Root, RootHex, allForks, Wei} from "@lodestar/types"; import {SLOTS_PER_EPOCH, ForkName, ForkSeq} from "@lodestar/params"; import {ErrorJsonRpcResponse, HttpRpcError} from "../../eth1/provider/jsonRpcHttpClient.js"; @@ -25,6 +25,7 @@ import { serializeExecutionPayload, serializeVersionedHashes, serializePayloadAttributes, + serializeBeaconBlockRoot, ExecutionPayloadBody, assertReqSizeLimit, deserializeExecutionPayloadBody, @@ -138,7 +139,8 @@ export class ExecutionEngineHttp implements IExecutionEngine { async notifyNewPayload( fork: ForkName, executionPayload: allForks.ExecutionPayload, - versionedHashes?: VersionedHashes + versionedHashes?: VersionedHashes, + parentBlockRoot?: Root ): Promise { const method = ForkSeq[fork] >= ForkSeq.deneb @@ -148,18 +150,23 @@ export class ExecutionEngineHttp implements IExecutionEngine { : "engine_newPayloadV1"; const serializedExecutionPayload = serializeExecutionPayload(fork, executionPayload); - const serializedVersionedHashes = - versionedHashes !== undefined ? serializeVersionedHashes(versionedHashes) : undefined; let engingRequest: EngineRequest; if (ForkSeq[fork] >= ForkSeq.deneb) { - if (serializedVersionedHashes === undefined) { + if (versionedHashes === undefined) { throw Error(`versionedHashes required in notifyNewPayload for fork=${fork}`); } + if (parentBlockRoot === undefined) { + throw Error(`parentBlockRoot required in notifyNewPayload for fork=${fork}`); + } + + const serializedVersionedHashes = serializeVersionedHashes(versionedHashes); + const parentBeaconBlockRoot = serializeBeaconBlockRoot(parentBlockRoot); + const method = "engine_newPayloadV3"; engingRequest = { method, - params: [serializedExecutionPayload, serializedVersionedHashes], + params: [serializedExecutionPayload, serializedVersionedHashes, parentBeaconBlockRoot], methodOpts: notifyNewPayloadOpts, }; } else { diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index d1bea600f5f3..c77c80a22e54 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -1,6 +1,6 @@ import {ForkName} from "@lodestar/params"; import {KZGCommitment, Blob, KZGProof} from "@lodestar/types/deneb"; -import {RootHex, allForks, capella, Wei} from "@lodestar/types"; +import {Root, RootHex, allForks, capella, Wei} from "@lodestar/types"; import {DATA, QUANTITY} from "../../eth1/provider/utils.js"; import {PayloadIdCache, PayloadId, WithdrawalV1} from "./payloadIdCache.js"; @@ -52,6 +52,7 @@ export type PayloadAttributes = { // avoid any conversions suggestedFeeRecipient: string; withdrawals?: capella.Withdrawal[]; + parentBeaconBlockRoot?: Uint8Array; }; export type TransitionConfigurationV1 = { @@ -92,7 +93,8 @@ export interface IExecutionEngine { notifyNewPayload( fork: ForkName, executionPayload: allForks.ExecutionPayload, - versionedHashes?: VersionedHashes + versionedHashes?: VersionedHashes, + parentBeaconBlockRoot?: Root ): Promise; /** diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 50608b3155e1..55f9e07b737a 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -1,4 +1,4 @@ -import {allForks, capella, deneb, Wei, bellatrix} from "@lodestar/types"; +import {allForks, capella, deneb, Wei, bellatrix, Root} from "@lodestar/types"; import { BYTES_PER_LOGS_BLOOM, FIELD_ELEMENTS_PER_BLOB, @@ -33,7 +33,7 @@ export type EngineApiRpcParamTypes = { */ engine_newPayloadV1: [ExecutionPayloadRpc]; engine_newPayloadV2: [ExecutionPayloadRpc]; - engine_newPayloadV3: [ExecutionPayloadRpc, VersionedHashesRpc]; + engine_newPayloadV3: [ExecutionPayloadRpc, VersionedHashesRpc, DATA]; /** * 1. Object - Payload validity status with respect to the consensus rules: * - blockHash: DATA, 32 Bytes - block hash value of the payload @@ -283,6 +283,10 @@ export function serializePayloadAttributes(data: PayloadAttributes): PayloadAttr }; } +export function serializeBeaconBlockRoot(data: Root): DATA { + return bytesToData(data); +} + export function deserializePayloadAttributes(data: PayloadAttributesRpc): PayloadAttributes { return { timestamp: quantityToNum(data.timestamp), diff --git a/packages/types/src/allForks/sszTypes.ts b/packages/types/src/allForks/sszTypes.ts index 49610ae476f1..8c0397ddf2c2 100644 --- a/packages/types/src/allForks/sszTypes.ts +++ b/packages/types/src/allForks/sszTypes.ts @@ -83,7 +83,7 @@ export const allForksExecution = { ExecutionPayloadHeader: deneb.ExecutionPayloadHeader, BuilderBid: deneb.BuilderBid, SignedBuilderBid: deneb.SignedBuilderBid, - SSEPayloadAttributes: capella.SSEPayloadAttributes, + SSEPayloadAttributes: deneb.SSEPayloadAttributes, }, }; diff --git a/packages/types/src/allForks/types.ts b/packages/types/src/allForks/types.ts index c4a76ff6a884..7f3e0f14e9e8 100644 --- a/packages/types/src/allForks/types.ts +++ b/packages/types/src/allForks/types.ts @@ -94,7 +94,10 @@ export type LightClientStore = altair.LightClientStore | capella.LightClientStor export type SignedBeaconBlockAndBlobsSidecar = deneb.SignedBeaconBlockAndBlobsSidecar; -export type SSEPayloadAttributes = bellatrix.SSEPayloadAttributes | capella.SSEPayloadAttributes; +export type SSEPayloadAttributes = + | bellatrix.SSEPayloadAttributes + | capella.SSEPayloadAttributes + | deneb.SSEPayloadAttributes; /** * Types known to change between forks */ @@ -222,7 +225,9 @@ export type AllForksExecutionSSZTypes = { typeof bellatrixSsz.SignedBuilderBid | typeof capellaSsz.SignedBuilderBid | typeof denebSsz.SignedBuilderBid >; SSEPayloadAttributes: AllForksTypeOf< - typeof bellatrixSsz.SSEPayloadAttributes | typeof capellaSsz.SSEPayloadAttributes + | typeof bellatrixSsz.SSEPayloadAttributes + | typeof capellaSsz.SSEPayloadAttributes + | typeof denebSsz.SSEPayloadAttributes >; }; diff --git a/packages/types/src/bellatrix/sszTypes.ts b/packages/types/src/bellatrix/sszTypes.ts index 5604ecf72ab6..8677c2a7c30e 100644 --- a/packages/types/src/bellatrix/sszTypes.ts +++ b/packages/types/src/bellatrix/sszTypes.ts @@ -9,6 +9,7 @@ import { import {ssz as primitiveSsz} from "../primitive/index.js"; import {ssz as phase0Ssz} from "../phase0/index.js"; import {ssz as altairSsz} from "../altair/index.js"; +import {stringType} from "../utils/StringType.js"; const { Bytes32, @@ -218,7 +219,7 @@ export const SignedBuilderBid = new ContainerType( // PayloadAttributes primarily for SSE event export const PayloadAttributes = new ContainerType( - {timestamp: UintNum64, prevRandao: Bytes32, suggestedFeeRecipient: ExecutionAddress}, + {timestamp: UintNum64, prevRandao: Bytes32, suggestedFeeRecipient: stringType}, {typeName: "PayloadAttributes", jsonCase: "eth2"} ); diff --git a/packages/types/src/deneb/sszTypes.ts b/packages/types/src/deneb/sszTypes.ts index 764170522d67..70101a0135ac 100644 --- a/packages/types/src/deneb/sszTypes.ts +++ b/packages/types/src/deneb/sszTypes.ts @@ -14,6 +14,7 @@ import { import {ssz as primitiveSsz} from "../primitive/index.js"; import {ssz as phase0Ssz} from "../phase0/index.js"; import {ssz as altairSsz} from "../altair/index.js"; +import {ssz as bellatrixSsz} from "../bellatrix/index.js"; import {ssz as capellaSsz} from "../capella/index.js"; const { @@ -385,3 +386,20 @@ export const LightClientStore = new ContainerType( }, {typeName: "LightClientStore", jsonCase: "eth2"} ); + +// PayloadAttributes primarily for SSE event +export const PayloadAttributes = new ContainerType( + { + ...capellaSsz.PayloadAttributes.fields, + parentBeaconBlockRoot: Root, + }, + {typeName: "PayloadAttributes", jsonCase: "eth2"} +); + +export const SSEPayloadAttributes = new ContainerType( + { + ...bellatrixSsz.SSEPayloadAttributesCommon.fields, + payloadAttributes: PayloadAttributes, + }, + {typeName: "SSEPayloadAttributes", jsonCase: "eth2"} +); diff --git a/packages/types/src/deneb/types.ts b/packages/types/src/deneb/types.ts index c804fef02f5c..a4d2286af1e6 100644 --- a/packages/types/src/deneb/types.ts +++ b/packages/types/src/deneb/types.ts @@ -48,6 +48,7 @@ export type FullOrBlindedExecutionPayload = ExecutionPayload | ExecutionPayloadH export type BuilderBid = ValueOf; export type SignedBuilderBid = ValueOf; +export type SSEPayloadAttributes = ValueOf; export type LightClientHeader = ValueOf; export type LightClientBootstrap = ValueOf; From 6638ea7ad20bd8a115b77503defbefd856a2379a Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Mon, 26 Jun 2023 21:29:57 +0700 Subject: [PATCH 27/96] fix: do not warn out-of-synced status at start up (#5708) --- packages/beacon-node/src/sync/sync.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/sync/sync.ts b/packages/beacon-node/src/sync/sync.ts index 7d56c1a27883..914d511eb8e0 100644 --- a/packages/beacon-node/src/sync/sync.ts +++ b/packages/beacon-node/src/sync/sync.ts @@ -223,8 +223,8 @@ export class BeaconSync implements IBeaconSync { else if (state !== SyncState.Synced) { const syncDiff = this.chain.clock.currentSlot - this.chain.forkChoice.getHead().slot; if (syncDiff > this.slotImportTolerance * 2) { - this.logger.warn(`Node sync has fallen behind by ${syncDiff} slots`); if (this.network.isSubscribedToGossipCoreTopics()) { + this.logger.warn(`Node sync has fallen behind by ${syncDiff} slots`); this.network .unsubscribeGossipCoreTopics() .then(() => { From 6be401d70407caa0353691f90f9965bb64c7071c Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 26 Jun 2023 19:31:16 +0300 Subject: [PATCH 28/96] fix: prevEpochBlockProposalSummary typo (#5707) Fix prevEpochBlockProposalSummary typo --- packages/beacon-node/src/metrics/validatorMonitor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/metrics/validatorMonitor.ts b/packages/beacon-node/src/metrics/validatorMonitor.ts index 8cd494f47017..ee51f24bd114 100644 --- a/packages/beacon-node/src/metrics/validatorMonitor.ts +++ b/packages/beacon-node/src/metrics/validatorMonitor.ts @@ -632,7 +632,7 @@ export function createValidatorMonitor( if (validator) { // If expected proposer is a tracked validator const summary = validator.summaries.get(prevEpoch); - metrics.validatorMonitor.prevEpochAttestationSummary.inc({ + metrics.validatorMonitor.prevEpochBlockProposalSummary.inc({ summary: renderBlockProposalSummary(config, rootCache, summary, SLOTS_PER_EPOCH * prevEpoch + slotIndex), }); } From ad971c68036c2bbe037c3a068f882aedca309ed2 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 26 Jun 2023 18:37:09 +0200 Subject: [PATCH 29/96] refactor: clean up validator import options (#5689) --- packages/cli/src/cmds/validator/import.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/cmds/validator/import.ts b/packages/cli/src/cmds/validator/import.ts index 75b083c95ce0..a39dfcc16f74 100644 --- a/packages/cli/src/cmds/validator/import.ts +++ b/packages/cli/src/cmds/validator/import.ts @@ -10,7 +10,11 @@ import {PersistedKeysBackend} from "./keymanager/persistedKeys.js"; /* eslint-disable no-console */ -export const importCmd: CliCommand = { +type ValidatorImportArgs = Pick; + +const {importKeystores, importKeystoresPassword} = validatorOptions; + +export const importCmd: CliCommand = { command: "import", describe: @@ -29,12 +33,11 @@ Ethereum Foundation utility.", // Note: re-uses `--importKeystores` and `--importKeystoresPassword` from root validator command options options: { - ...validatorOptions, - importKeystores: { - ...validatorOptions.importKeystores, + ...importKeystores, requiresArg: true, }, + importKeystoresPassword, }, handler: async (args) => { From a208afb45a504960c3897831b2e446083e43c0a3 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 26 Jun 2023 19:59:00 +0300 Subject: [PATCH 30/96] feat: emit eventstream events with incoming gossip (#5596) Emit eventstream events with incoming gossip --- .../src/api/impl/beacon/pool/index.ts | 7 ++++ .../src/chain/blocks/importBlock.ts | 34 ++++++++++++------- .../src/network/processor/gossipHandlers.ts | 13 +++++++ 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index 60f4539adc01..15bb1a3107db 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -70,6 +70,9 @@ export function getBeaconPoolApi({ const insertOutcome = chain.attestationPool.add(attestation, attDataRootHex); metrics?.opPool.attestationPoolInsertOutcome.inc({insertOutcome}); } + + chain.emitter.emit(routes.events.EventType.attestation, attestation); + const sentPeers = await network.publishBeaconAttestation(attestation, subnet); metrics?.onPoolSubmitUnaggregatedAttestation(seenTimestampSec, indexedAttestation, subnet, sentPeers); } catch (e) { @@ -108,6 +111,7 @@ export function getBeaconPoolApi({ async submitPoolVoluntaryExit(voluntaryExit) { await validateGossipVoluntaryExit(chain, voluntaryExit); chain.opPool.insertVoluntaryExit(voluntaryExit); + chain.emitter.emit(routes.events.EventType.voluntaryExit, voluntaryExit); await network.publishVoluntaryExit(voluntaryExit); }, @@ -121,6 +125,9 @@ export function getBeaconPoolApi({ await validateBlsToExecutionChange(chain, blsToExecutionChange, true); const preCapella = chain.clock.currentEpoch < chain.config.CAPELLA_FORK_EPOCH; chain.opPool.insertBlsToExecutionChange(blsToExecutionChange, preCapella); + + chain.emitter.emit(routes.events.EventType.blsToExecutionChange, blsToExecutionChange); + if (!preCapella) { await network.publishBlsToExecutionChange(blsToExecutionChange); } diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 02789ece50cb..8da566b9d2bb 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -26,6 +26,10 @@ import {writeBlockInputToDb} from "./writeBlockInputToDb.js"; * Fork-choice allows to import attestations from current (0) or past (1) epoch. */ const FORK_CHOICE_ATT_EPOCH_LIMIT = 1; +/** + * Emit eventstream events for block contents events only for blocks that are recent enough to clock + */ +const EVENTSTREAM_EMIT_RECENT_BLOCK_SLOTS = 64; /** * Imports a fully verified block into the chain state. Produces multiple permanent side-effects. @@ -149,11 +153,6 @@ export async function importBlock( blockRootHex, block.message.slot ); - - // don't want to log the processed attestations here as there are so many attestations and it takes too much disc space, - // users may want to keep more log files instead of unnecessary processed attestations log - // see https://github.com/ChainSafe/lodestar/pull/4032 - this.emitter.emit(routes.events.EventType.attestation, attestation); } catch (e) { // a block has a lot of attestations and it may has same error, we don't want to log all of them if (e instanceof ForkChoiceError && e.type.code === ForkChoiceErrorCode.INVALID_ATTESTATION) { @@ -370,14 +369,25 @@ export async function importBlock( } } - // Send block events + // Send block events, only for recent enough blocks - for (const voluntaryExit of block.message.body.voluntaryExits) { - this.emitter.emit(routes.events.EventType.voluntaryExit, voluntaryExit); - } - - for (const blsToExecutionChange of (block.message.body as capella.BeaconBlockBody).blsToExecutionChanges ?? []) { - this.emitter.emit(routes.events.EventType.blsToExecutionChange, blsToExecutionChange); + if (this.clock.currentSlot - block.message.slot < EVENTSTREAM_EMIT_RECENT_BLOCK_SLOTS) { + // NOTE: Skip looping if there are no listeners from the API + if (this.emitter.listenerCount(routes.events.EventType.voluntaryExit)) { + for (const voluntaryExit of block.message.body.voluntaryExits) { + this.emitter.emit(routes.events.EventType.voluntaryExit, voluntaryExit); + } + } + if (this.emitter.listenerCount(routes.events.EventType.blsToExecutionChange)) { + for (const blsToExecutionChange of (block.message.body as capella.BeaconBlockBody).blsToExecutionChanges ?? []) { + this.emitter.emit(routes.events.EventType.blsToExecutionChange, blsToExecutionChange); + } + } + if (this.emitter.listenerCount(routes.events.EventType.attestation)) { + for (const attestation of block.message.body.attestations) { + this.emitter.emit(routes.events.EventType.attestation, attestation); + } + } } // Register stat metrics about the block after importing it diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 06050cfa0ff5..4399bda0b004 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -3,6 +3,7 @@ import {BeaconConfig} from "@lodestar/config"; import {Logger, prettyBytes} from "@lodestar/utils"; import {Root, Slot, ssz} from "@lodestar/types"; import {ForkName, ForkSeq} from "@lodestar/params"; +import {routes} from "@lodestar/api"; import {Metrics} from "../../metrics/index.js"; import {OpSource} from "../../metrics/validatorMonitor.js"; import {IBeaconChain} from "../../chain/index.js"; @@ -177,6 +178,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH const blockInput = getBlockInput.preDeneb(config, signedBlock, BlockSource.gossip, serializedData); await validateBeaconBlock(blockInput, topic.fork, peerIdStr, seenTimestampSec); handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); + + // Do not emit block on eventstream API, it will be emitted after successful import }, [GossipType.blob_sidecar]: async (_data, _topic, _peerIdStr, _seenTimestampSec) => { @@ -235,6 +238,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH ); } } + + chain.emitter.emit(routes.events.EventType.attestation, signedAggregateAndProof.message.aggregate); }, [GossipType.beacon_attestation]: async ({serializedData, msgSlot}, {subnet}, _peer, seenTimestampSec) => { @@ -278,6 +283,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH logger.debug("Error adding gossip unaggregated attestation to forkchoice", {subnet}, e as Error); } } + + chain.emitter.emit(routes.events.EventType.attestation, attestation); }, [GossipType.attester_slashing]: async ({serializedData}, topic) => { @@ -318,6 +325,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } catch (e) { logger.error("Error adding voluntaryExit to pool", {}, e as Error); } + + chain.emitter.emit(routes.events.EventType.voluntaryExit, voluntaryExit); }, [GossipType.sync_committee_contribution_and_proof]: async ({serializedData}, topic) => { @@ -340,6 +349,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } catch (e) { logger.error("Error adding to contributionAndProof pool", {}, e as Error); } + + chain.emitter.emit(routes.events.EventType.contributionAndProof, contributionAndProof); }, [GossipType.sync_committee]: async ({serializedData}, topic) => { @@ -386,6 +397,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } catch (e) { logger.error("Error adding blsToExecutionChange to pool", {}, e as Error); } + + chain.emitter.emit(routes.events.EventType.blsToExecutionChange, blsToExecutionChange); }, }; } From 00877eec1c0ddc5fa4662a430a71e180d4996f5d Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Tue, 27 Jun 2023 00:50:45 -0700 Subject: [PATCH 31/96] docs: correction on weak subjectivity (#5712) --- docs/usage/beacon-management.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/usage/beacon-management.md b/docs/usage/beacon-management.md index 34709ebcd7f6..5536fbb78a5b 100644 --- a/docs/usage/beacon-management.md +++ b/docs/usage/beacon-management.md @@ -101,8 +101,7 @@ A young testnet should take a few hours to sync. If you see multiple or consiste ### Checkpoint Sync -If you are starting your node from a blank db/genesis (or from last saved state in db) in a network which is now far ahead, your node is susceptible to "long range attacks" via something called weak subjectivity. -[Read Vitalik's illuminating post on the same](https://blog.ethereum.org/2014/11/25/proof-stake-learned-love-weak-subjectivity/). +If you are starting your node from a blank db, like starting from genesis, or from the last saved state in db and the network is now far ahead, your node will be susceptible to "long range attacks." Ethereum's solution to this is via something called weak subjectivity. [Read Vitalik's illuminating post explaining weak subjectivity.](https://blog.ethereum.org/2014/11/25/proof-stake-learned-love-weak-subjectivity/). If you have a synced beacon node available (e.g., your friend's node or an infrastructure provider) and a trusted checkpoint you can rely on, you can start off your beacon node in under a minute! And at the same time kicking the "long range attack" in its butt! @@ -182,4 +181,4 @@ Apr-20 15:16:17.017[] info: Synced - slot: 6264979 - head: 0xde9 6. Peer info: Current total number of outbound or inbound peers, for e.g.: `peers: 27` -For more insight into lodestar beacon functioning, you may setup lodestar metrics and use prepared Grafana dashboards that you may find in the repository. +For more insight into how a Lodestar beacon node is functioning, you may setup lodestar metrics and use the prepared Grafana dashboards that are found in the repository. Check out our section on [Prometheus and Grafana](./prometheus-grafana.md) for more details. From 158d8d968f04e028f6e21d9804b23139e34b8198 Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Tue, 27 Jun 2023 19:22:08 +0700 Subject: [PATCH 32/96] fix: correct subscribedToCoreTopics flag (#5715) --- packages/beacon-node/src/network/network.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index 3f3adb39076a..f5f807796356 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -266,7 +266,8 @@ export class Network implements INetwork { // Drop all the gossip validation queues this.networkProcessor.dropAllJobs(); - return this.core.unsubscribeGossipCoreTopics(); + await this.core.unsubscribeGossipCoreTopics(); + this.subscribedToCoreTopics = false; } isSubscribedToGossipCoreTopics(): boolean { From 2f687cf5b17f9c7b035e52e4e2349c7adb1603a9 Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Tue, 27 Jun 2023 19:23:30 +0700 Subject: [PATCH 33/96] fix: ignore ALREADY_KNOWN gossip block error (#5714) --- packages/beacon-node/src/network/processor/gossipHandlers.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 4399bda0b004..ed567b846209 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -128,6 +128,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH chain .processBlock(blockInput, { + // block may be downloaded and processed by UnknownBlockSync + ignoreIfKnown: true, // proposer signature already checked in validateBeaconBlock() validProposerSignature: true, // blobsSidecar already checked in validateGossipBlobsSidecar() @@ -152,6 +154,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH .catch((e) => { if (e instanceof BlockError) { switch (e.type.code) { + // ALREADY_KNOWN should not happen with ignoreIfKnown=true above + // PARENT_UNKNOWN should not happen, we handled this in validateBeaconBlock() function above case BlockErrorCode.ALREADY_KNOWN: case BlockErrorCode.PARENT_UNKNOWN: case BlockErrorCode.PRESTATE_MISSING: From d9fa3555f862138dcfbf3b21a7fe61e16a9a9e88 Mon Sep 17 00:00:00 2001 From: g11tech Date: Tue, 27 Jun 2023 17:58:49 +0530 Subject: [PATCH 34/96] feat: migrate to blob side cars validation from blobs side car (#5687) * feat: migrate to blob side cars validation from blobs side car * fix test * network test fix * apply feedback --- .../beacon-node/src/chain/blocks/types.ts | 2 +- .../chain/blocks/verifyBlocksSanityChecks.ts | 9 +- ...obsSidecarError.ts => blobSidecarError.ts} | 18 +- .../beacon-node/src/chain/errors/index.ts | 2 +- .../beacon-node/src/chain/regen/interface.ts | 1 + .../src/chain/validation/blobSidecar.ts | 198 ++++++++++++++++++ .../src/chain/validation/blobsSidecar.ts | 143 ------------- .../src/metrics/validatorMonitor.ts | 7 +- .../src/network/gossip/interface.ts | 6 - .../beacon-node/src/network/gossip/topic.ts | 18 +- .../network/processor/extractSlotRootFns.ts | 6 +- .../src/network/processor/gossipHandlers.ts | 21 +- .../src/network/processor/gossipQueues.ts | 6 - .../src/network/processor/index.ts | 3 +- packages/beacon-node/src/util/sszBytes.ts | 33 ++- .../test/e2e/network/network.test.ts | 2 +- .../test/spec/presets/fork_choice.ts | 2 +- .../test/unit/network/gossip/topic.test.ts | 6 - .../beacon-node/test/unit/util/kzg.test.ts | 8 +- .../test/unit/util/sszBytes.test.ts | 33 ++- .../beacon-node/test/utils/node/beacon.ts | 6 +- packages/types/src/deneb/sszTypes.ts | 1 + packages/types/src/deneb/types.ts | 1 + 23 files changed, 280 insertions(+), 252 deletions(-) rename packages/beacon-node/src/chain/errors/{blobsSidecarError.ts => blobSidecarError.ts} (54%) create mode 100644 packages/beacon-node/src/chain/validation/blobSidecar.ts delete mode 100644 packages/beacon-node/src/chain/validation/blobsSidecar.ts diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index affbbbe5fd9e..1d62305642f7 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -128,7 +128,7 @@ export type ImportBlockOpts = { */ validSignatures?: boolean; /** Set to true if already run `validateBlobsSidecar()` sucessfully on the blobs */ - validBlobsSidecar?: boolean; + validBlobSidecars?: boolean; /** Seen timestamp seconds */ seenTimestampSec?: number; /** Set to true if persist block right at verification time */ diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts index 7fa5bb6fa405..c0ea81fbb173 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts @@ -5,7 +5,9 @@ import {Slot, deneb} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {BlockError, BlockErrorCode} from "../errors/index.js"; -import {validateBlobsSidecar} from "../validation/blobsSidecar.js"; +// TODO freetheblobs: disable the following exception once blockinput changes +/* eslint-disable @typescript-eslint/no-unused-vars */ +import {validateBlobSidecars} from "../validation/blobSidecar.js"; import {BlockInput, BlockInputType, ImportBlockOpts} from "./types.js"; /** @@ -126,7 +128,7 @@ function maybeValidateBlobs( // TODO Deneb: Make switch verify it's exhaustive switch (blockInput.type) { case BlockInputType.postDeneb: { - if (opts.validBlobsSidecar) { + if (opts.validBlobSidecars) { return DataAvailableStatus.available; } @@ -135,7 +137,8 @@ function maybeValidateBlobs( const {blobKzgCommitments} = (block as deneb.SignedBeaconBlock).message.body; const beaconBlockRoot = config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message); // TODO Deneb: This function throws un-typed errors - validateBlobsSidecar(blockSlot, beaconBlockRoot, blobKzgCommitments, blobs); + // TODO freetheblobs: enable the following validation once blockinput is migrated + // validateBlobSidecars(blockSlot, beaconBlockRoot, blobKzgCommitments, blobs); return DataAvailableStatus.available; } diff --git a/packages/beacon-node/src/chain/errors/blobsSidecarError.ts b/packages/beacon-node/src/chain/errors/blobSidecarError.ts similarity index 54% rename from packages/beacon-node/src/chain/errors/blobsSidecarError.ts rename to packages/beacon-node/src/chain/errors/blobSidecarError.ts index 628a2cdc0640..f0ad69167c19 100644 --- a/packages/beacon-node/src/chain/errors/blobsSidecarError.ts +++ b/packages/beacon-node/src/chain/errors/blobSidecarError.ts @@ -1,7 +1,8 @@ import {Slot} from "@lodestar/types"; import {GossipActionError} from "./gossipValidation.js"; -export enum BlobsSidecarErrorCode { +export enum BlobSidecarErrorCode { + INVALID_INDEX = "BLOBS_SIDECAR_ERROR_INVALID_INDEX", /** !bls.KeyValidate(block.body.blob_kzg_commitments[i]) */ INVALID_KZG = "BLOBS_SIDECAR_ERROR_INVALID_KZG", /** !verify_kzg_commitments_against_transactions(block.body.execution_payload.transactions, block.body.blob_kzg_commitments) */ @@ -14,11 +15,12 @@ export enum BlobsSidecarErrorCode { INVALID_KZG_PROOF = "BLOBS_SIDECAR_ERROR_INVALID_KZG_PROOF", } -export type BlobsSidecarErrorType = - | {code: BlobsSidecarErrorCode.INVALID_KZG; kzgIdx: number} - | {code: BlobsSidecarErrorCode.INVALID_KZG_TXS} - | {code: BlobsSidecarErrorCode.INCORRECT_SLOT; blockSlot: Slot; blobSlot: Slot} - | {code: BlobsSidecarErrorCode.INVALID_BLOB; blobIdx: number} - | {code: BlobsSidecarErrorCode.INVALID_KZG_PROOF}; +export type BlobSidecarErrorType = + | {code: BlobSidecarErrorCode.INVALID_INDEX; blobIdx: number; gossipIndex: number} + | {code: BlobSidecarErrorCode.INVALID_KZG; blobIdx: number} + | {code: BlobSidecarErrorCode.INVALID_KZG_TXS} + | {code: BlobSidecarErrorCode.INCORRECT_SLOT; blockSlot: Slot; blobSlot: Slot; blobIdx: number} + | {code: BlobSidecarErrorCode.INVALID_BLOB; blobIdx: number} + | {code: BlobSidecarErrorCode.INVALID_KZG_PROOF; blobIdx: number}; -export class BlobsSidecarError extends GossipActionError {} +export class BlobSidecarError extends GossipActionError {} diff --git a/packages/beacon-node/src/chain/errors/index.ts b/packages/beacon-node/src/chain/errors/index.ts index 9ce6c8109749..1bd8f8577305 100644 --- a/packages/beacon-node/src/chain/errors/index.ts +++ b/packages/beacon-node/src/chain/errors/index.ts @@ -1,6 +1,6 @@ export * from "./attestationError.js"; export * from "./attesterSlashingError.js"; -export * from "./blobsSidecarError.js"; +export * from "./blobSidecarError.js"; export * from "./blockError.js"; export * from "./gossipValidation.js"; export * from "./proposerSlashingError.js"; diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index a0989f30c41f..e7be64d0eecb 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -9,6 +9,7 @@ export enum RegenCaller { processBlock = "processBlock", produceBlock = "produceBlock", validateGossipBlock = "validateGossipBlock", + validateGossipBlob = "validateGossipBlob", precomputeEpoch = "precomputeEpoch", produceAttestationData = "produceAttestationData", processBlocksInEpoch = "processBlocksInEpoch", diff --git a/packages/beacon-node/src/chain/validation/blobSidecar.ts b/packages/beacon-node/src/chain/validation/blobSidecar.ts new file mode 100644 index 000000000000..9f51b29b817e --- /dev/null +++ b/packages/beacon-node/src/chain/validation/blobSidecar.ts @@ -0,0 +1,198 @@ +import {ChainForkConfig} from "@lodestar/config"; +import {deneb, Root, Slot} from "@lodestar/types"; +import {toHex} from "@lodestar/utils"; +import {getBlobProposerSignatureSet, computeStartSlotAtEpoch} from "@lodestar/state-transition"; + +import {BlobSidecarError, BlobSidecarErrorCode} from "../errors/blobSidecarError.js"; +import {GossipAction} from "../errors/gossipValidation.js"; +import {ckzg} from "../../util/kzg.js"; +import {byteArrayEquals} from "../../util/bytes.js"; +import {IBeaconChain} from "../interface.js"; +import {RegenCaller} from "../regen/index.js"; + +// TODO: freetheblobs define blobs own gossip error +import {BlockGossipError, BlockErrorCode} from "../errors/index.js"; + +export async function validateGossipBlobSidecar( + config: ChainForkConfig, + chain: IBeaconChain, + signedBlob: deneb.SignedBlobSidecar, + gossipIndex: number +): Promise { + const blobSidecar = signedBlob.message; + const blobSlot = blobSidecar.slot; + + // [REJECT] The sidecar is for the correct topic -- i.e. sidecar.index matches the topic {index}. + if (blobSidecar.index !== gossipIndex) { + throw new BlobSidecarError(GossipAction.REJECT, { + code: BlobSidecarErrorCode.INVALID_INDEX, + blobIdx: blobSidecar.index, + gossipIndex, + }); + } + + // [IGNORE] The sidecar is not from a future slot (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) -- + // i.e. validate that sidecar.slot <= current_slot (a client MAY queue future blocks for processing at + // the appropriate slot). + const currentSlotWithGossipDisparity = chain.clock.currentSlotWithGossipDisparity; + if (currentSlotWithGossipDisparity < blobSlot) { + throw new BlockGossipError(GossipAction.IGNORE, { + code: BlockErrorCode.FUTURE_SLOT, + currentSlot: currentSlotWithGossipDisparity, + blockSlot: blobSlot, + }); + } + + // [IGNORE] The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that + // sidecar.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) + const finalizedCheckpoint = chain.forkChoice.getFinalizedCheckpoint(); + const finalizedSlot = computeStartSlotAtEpoch(finalizedCheckpoint.epoch); + if (blobSlot <= finalizedSlot) { + throw new BlockGossipError(GossipAction.IGNORE, { + code: BlockErrorCode.WOULD_REVERT_FINALIZED_SLOT, + blockSlot: blobSlot, + finalizedSlot, + }); + } + + // Check if the block is already known. We know it is post-finalization, so it is sufficient to check the fork choice. + // + // In normal operation this isn't necessary, however it is useful immediately after a + // reboot if the `observed_block_producers` cache is empty. In that case, without this + // check, we will load the parent and state from disk only to find out later that we + // already know this block. + const blockRoot = toHex(blobSidecar.blockRoot); + if (chain.forkChoice.getBlockHex(blockRoot) !== null) { + throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.ALREADY_KNOWN, root: blockRoot}); + } + + // TODO: freetheblobs - check for badblock + // TODO: freetheblobs - check that its first blob with valid signature + + // _[IGNORE]_ The blob's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both + // gossip and non-gossip sources) (a client MAY queue blocks for processing once the parent block is + // retrieved). + const parentRoot = toHex(blobSidecar.blockParentRoot); + const parentBlock = chain.forkChoice.getBlockHex(parentRoot); + if (parentBlock === null) { + // If fork choice does *not* consider the parent to be a descendant of the finalized block, + // then there are two more cases: + // + // 1. We have the parent stored in our database. Because fork-choice has confirmed the + // parent is *not* in our post-finalization DAG, all other blocks must be either + // pre-finalization or conflicting with finalization. + // 2. The parent is unknown to us, we probably want to download it since it might actually + // descend from the finalized root. + // (Non-Lighthouse): Since we prune all blocks non-descendant from finalized checking the `db.block` database won't be useful to guard + // against known bad fork blocks, so we throw PARENT_UNKNOWN for cases (1) and (2) + throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.PARENT_UNKNOWN, parentRoot}); + } + + // [REJECT] The blob is from a higher slot than its parent. + if (parentBlock.slot >= blobSlot) { + throw new BlockGossipError(GossipAction.IGNORE, { + code: BlockErrorCode.NOT_LATER_THAN_PARENT, + parentSlot: parentBlock.slot, + slot: blobSlot, + }); + } + + // getBlockSlotState also checks for whether the current finalized checkpoint is an ancestor of the block. + // As a result, we throw an IGNORE (whereas the spec says we should REJECT for this scenario). + // this is something we should change this in the future to make the code airtight to the spec. + // _[IGNORE]_ The blob's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both + // gossip and non-gossip sources) // _[REJECT]_ The blob's block's parent (defined by `sidecar.block_parent_root`) passes validation + // The above validation will happen while importing + const blockState = await chain.regen + .getBlockSlotState(parentRoot, blobSlot, {dontTransferCache: true}, RegenCaller.validateGossipBlob) + .catch(() => { + throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.PARENT_UNKNOWN, parentRoot}); + }); + + // _[REJECT]_ The proposer signature, `signed_blob_sidecar.signature`, is valid with respect to the + // `sidecar.proposer_index` pubkey. + const signatureSet = getBlobProposerSignatureSet(blockState, signedBlob); + // Don't batch so verification is not delayed + if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) { + throw new BlockGossipError(GossipAction.REJECT, { + code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID, + }); + } + + // _[IGNORE]_ The sidecar is the only sidecar with valid signature received for the tuple + // `(sidecar.block_root, sidecar.index)` + // + // This is already taken care of by the way we group the blobs in getFullBlockInput helper + // but may be an error can be thrown there for this + + // _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the + // context of the current shuffling (defined by `block_parent_root`/`slot`) + // If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar + // MAY be queued for later processing while proposers for the block's branch are calculated -- in such + // a case _do not_ `REJECT`, instead `IGNORE` this message. + const proposerIndex = blobSidecar.proposerIndex; + if (blockState.epochCtx.getBeaconProposer(blobSlot) !== proposerIndex) { + throw new BlockGossipError(GossipAction.REJECT, {code: BlockErrorCode.INCORRECT_PROPOSER, proposerIndex}); + } + + // blob, proof and commitment as a valid BLS G1 point gets verified in batch validation + validateBlobsAndProofs([blobSidecar.kzgCommitment], [blobSidecar.blob], [blobSidecar.kzgProof]); +} + +// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#validate_blobs_sidecar +export function validateBlobSidecars( + blockSlot: Slot, + blockRoot: Root, + expectedKzgCommitments: deneb.BlobKzgCommitments, + blobSidecars: deneb.BlobSidecars +): void { + // assert len(expected_kzg_commitments) == len(blobs) + if (expectedKzgCommitments.length !== blobSidecars.length) { + throw new Error( + `blobSidecars length to commitments length mismatch. Blob length: ${blobSidecars.length}, Expected commitments length ${expectedKzgCommitments.length}` + ); + } + + // No need to verify the aggregate proof of zero blobs + if (blobSidecars.length > 0) { + // Verify the blob slot and root matches + const blobs = []; + const proofs = []; + for (let index = 0; index < blobSidecars.length; index++) { + const blobSidecar = blobSidecars[index]; + if ( + blobSidecar.slot !== blockSlot || + !byteArrayEquals(blobSidecar.blockRoot, blockRoot) || + blobSidecar.index !== index || + !byteArrayEquals(expectedKzgCommitments[index], blobSidecar.kzgCommitment) + ) { + throw new Error( + `Invalid blob with slot=${blobSidecar.slot} blockRoot=${toHex(blockRoot)} index=${ + blobSidecar.index + } for the block root=${toHex(blockRoot)} slot=${blockSlot} index=${index}` + ); + } + blobs.push(blobSidecar.blob); + proofs.push(blobSidecar.kzgProof); + } + validateBlobsAndProofs(expectedKzgCommitments, blobs, proofs); + } +} + +function validateBlobsAndProofs( + expectedKzgCommitments: deneb.BlobKzgCommitments, + blobs: deneb.Blobs, + proofs: deneb.KZGProofs +): void { + // assert verify_aggregate_kzg_proof(blobs, expected_kzg_commitments, kzg_aggregated_proof) + let isProofValid: boolean; + try { + isProofValid = ckzg.verifyBlobKzgProofBatch(blobs, expectedKzgCommitments, proofs); + } catch (e) { + (e as Error).message = `Error on verifyBlobKzgProofBatch: ${(e as Error).message}`; + throw e; + } + if (!isProofValid) { + throw Error("Invalid verifyBlobKzgProofBatch"); + } +} diff --git a/packages/beacon-node/src/chain/validation/blobsSidecar.ts b/packages/beacon-node/src/chain/validation/blobsSidecar.ts deleted file mode 100644 index ad5eddb8c9a6..000000000000 --- a/packages/beacon-node/src/chain/validation/blobsSidecar.ts +++ /dev/null @@ -1,143 +0,0 @@ -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/bls/types"; -import {deneb, Root, ssz} from "@lodestar/types"; -import {bytesToBigInt, toHex} from "@lodestar/utils"; -import {BYTES_PER_FIELD_ELEMENT, FIELD_ELEMENTS_PER_BLOB} from "@lodestar/params"; -import {BlobsSidecarError, BlobsSidecarErrorCode} from "../errors/blobsSidecarError.js"; -import {GossipAction} from "../errors/gossipValidation.js"; -import {byteArrayEquals} from "../../util/bytes.js"; -import {ckzg} from "../../util/kzg.js"; - -const BLS_MODULUS = BigInt("52435875175126190479447740508185965837690552500527637822603658699938581184513"); - -export function validateGossipBlobsSidecar( - signedBlock: deneb.SignedBeaconBlock, - blobsSidecar: deneb.BlobsSidecar -): void { - const block = signedBlock.message; - - // Spec: https://github.com/ethereum/consensus-specs/blob/4cb6fd1c8c8f190d147d15b182c2510d0423ec61/specs/eip4844/p2p-interface.md#beacon_block_and_blobs_sidecar - // [REJECT] The KZG commitments of the blobs are all correctly encoded compressed BLS G1 Points. - // -- i.e. all(bls.KeyValidate(commitment) for commitment in block.body.blob_kzg_commitments) - const {blobKzgCommitments} = block.body; - for (let i = 0; i < blobKzgCommitments.length; i++) { - if (!blsKeyValidate(blobKzgCommitments[i])) { - throw new BlobsSidecarError(GossipAction.REJECT, {code: BlobsSidecarErrorCode.INVALID_KZG, kzgIdx: i}); - } - } - - // [IGNORE] the sidecar.beacon_block_slot is for the current slot (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) - // -- i.e. sidecar.beacon_block_slot == block.slot. - if (blobsSidecar.beaconBlockSlot !== block.slot) { - throw new BlobsSidecarError(GossipAction.IGNORE, { - code: BlobsSidecarErrorCode.INCORRECT_SLOT, - blobSlot: blobsSidecar.beaconBlockSlot, - blockSlot: block.slot, - }); - } - - // [REJECT] the sidecar.blobs are all well formatted, i.e. the BLSFieldElement in valid range (x < BLS_MODULUS). - for (let i = 0; i < blobsSidecar.blobs.length; i++) { - if (!blobIsValidRange(blobsSidecar.blobs[i])) { - throw new BlobsSidecarError(GossipAction.REJECT, {code: BlobsSidecarErrorCode.INVALID_BLOB, blobIdx: i}); - } - } - - // [REJECT] The KZG proof is a correctly encoded compressed BLS G1 Point - // -- i.e. blsKeyValidate(blobs_sidecar.kzg_aggregated_proof) - if (!blsKeyValidate(blobsSidecar.kzgAggregatedProof)) { - throw new BlobsSidecarError(GossipAction.REJECT, {code: BlobsSidecarErrorCode.INVALID_KZG_PROOF}); - } - - // [REJECT] The KZG commitments in the block are valid against the provided blobs sidecar. -- i.e. - // validate_blobs_sidecar(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments, sidecar) - validateBlobsSidecar( - block.slot, - ssz.bellatrix.BeaconBlock.hashTreeRoot(block), - block.body.blobKzgCommitments, - blobsSidecar - ); -} - -// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#validate_blobs_sidecar -export function validateBlobsSidecar( - slot: number, - beaconBlockRoot: Root, - expectedKzgCommitments: deneb.KZGCommitment[], - blobsSidecar: deneb.BlobsSidecar -): void { - // assert slot == blobs_sidecar.beacon_block_slot - if (slot != blobsSidecar.beaconBlockSlot) { - throw new Error(`slot mismatch. Block slot: ${slot}, Blob slot ${blobsSidecar.beaconBlockSlot}`); - } - - // assert beacon_block_root == blobs_sidecar.beacon_block_root - if (!byteArrayEquals(beaconBlockRoot, blobsSidecar.beaconBlockRoot)) { - throw new Error( - `beacon block root mismatch. Block root: ${toHex(beaconBlockRoot)}, Blob root ${toHex( - blobsSidecar.beaconBlockRoot - )}` - ); - } - - // blobs = blobs_sidecar.blobs - // kzg_aggregated_proof = blobs_sidecar.kzg_aggregated_proof - const {blobs, kzgAggregatedProof} = blobsSidecar; - - // assert len(expected_kzg_commitments) == len(blobs) - if (expectedKzgCommitments.length !== blobs.length) { - throw new Error( - `blobs length to commitments length mismatch. Blob length: ${blobs.length}, Expected commitments length ${expectedKzgCommitments.length}` - ); - } - - // No need to verify the aggregate proof of zero blobs. Also c-kzg throws. - // https://github.com/dankrad/c-kzg/pull/12/files#r1025851956 - if (blobs.length > 0) { - // assert verify_aggregate_kzg_proof(blobs, expected_kzg_commitments, kzg_aggregated_proof) - let isProofValid: boolean; - try { - isProofValid = ckzg.verifyAggregateKzgProof(blobs, expectedKzgCommitments, kzgAggregatedProof); - } catch (e) { - // TODO DENEB: TEMP Nov17: May always throw error -- we need to fix Geth's KZG to match C-KZG and the trusted setup used here - (e as Error).message = `Error on verifyAggregateKzgProof: ${(e as Error).message}`; - throw e; - } - - // TODO DENEB: TEMP Nov17: May always throw error -- we need to fix Geth's KZG to match C-KZG and the trusted setup used here - if (!isProofValid) { - throw Error("Invalid AggregateKzgProof"); - } - } -} - -/** - * From https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-bls-signature-04#section-2.5 - * KeyValidate = valid, non-identity point that is in the correct subgroup - */ -function blsKeyValidate(g1Point: Uint8Array): boolean { - try { - bls.PublicKey.fromBytes(g1Point, CoordType.jacobian, true); - return true; - } catch (e) { - return false; - } -} - -/** - * ``` - * Blob = new ByteVectorType(BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB); - * ``` - * Check that each FIELD_ELEMENT as a uint256 < BLS_MODULUS - */ -function blobIsValidRange(blob: deneb.Blob): boolean { - for (let i = 0; i < FIELD_ELEMENTS_PER_BLOB; i++) { - const fieldElement = blob.subarray(i * BYTES_PER_FIELD_ELEMENT, (i + 1) * BYTES_PER_FIELD_ELEMENT); - const fieldElementBN = bytesToBigInt(fieldElement, "be"); - if (fieldElementBN >= BLS_MODULUS) { - return false; - } - } - - return true; -} diff --git a/packages/beacon-node/src/metrics/validatorMonitor.ts b/packages/beacon-node/src/metrics/validatorMonitor.ts index ee51f24bd114..5b0a5e1d5089 100644 --- a/packages/beacon-node/src/metrics/validatorMonitor.ts +++ b/packages/beacon-node/src/metrics/validatorMonitor.ts @@ -10,7 +10,7 @@ import { ParticipationFlags, } from "@lodestar/state-transition"; import {Logger, MapDef, MapDefMax, toHex} from "@lodestar/utils"; -import {RootHex, allForks, altair} from "@lodestar/types"; +import {RootHex, allForks, altair, deneb} from "@lodestar/types"; import {ChainConfig, ChainForkConfig} from "@lodestar/config"; import {ForkSeq, INTERVALS_PER_SLOT, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; import {Epoch, Slot, ValidatorIndex} from "@lodestar/types"; @@ -40,6 +40,7 @@ export type ValidatorMonitor = { registerLocalValidatorInSyncCommittee(index: number, untilEpoch: Epoch): void; registerValidatorStatuses(currentEpoch: Epoch, statuses: AttesterStatus[], balances?: number[]): void; registerBeaconBlock(src: OpSource, seenTimestampSec: Seconds, block: allForks.BeaconBlock): void; + registerBlobSidecar(src: OpSource, seenTimestampSec: Seconds, blob: deneb.BlobSidecar): void; registerImportedBlock(block: allForks.BeaconBlock, data: {proposerBalanceDelta: number}): void; onPoolSubmitUnaggregatedAttestation( seenTimestampSec: number, @@ -376,6 +377,10 @@ export function createValidatorMonitor( } }, + registerBlobSidecar(_src, _seenTimestampSec, _blob) { + //TODO: freetheblobs + }, + registerImportedBlock(block, {proposerBalanceDelta}) { const validator = validators.get(block.proposerIndex); if (validator) { diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index dc178e8ee246..6f1365a77c9c 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -11,7 +11,6 @@ import {JobItemQueue} from "../../util/queue/index.js"; export enum GossipType { beacon_block = "beacon_block", blob_sidecar = "blob_sidecar", - beacon_block_and_blobs_sidecar = "beacon_block_and_blobs_sidecar", beacon_aggregate_and_proof = "beacon_aggregate_and_proof", beacon_attestation = "beacon_attestation", voluntary_exit = "voluntary_exit", @@ -40,7 +39,6 @@ export interface IGossipTopic { export type GossipTopicTypeMap = { [GossipType.beacon_block]: {type: GossipType.beacon_block}; [GossipType.blob_sidecar]: {type: GossipType.blob_sidecar; index: number}; - [GossipType.beacon_block_and_blobs_sidecar]: {type: GossipType.beacon_block_and_blobs_sidecar}; [GossipType.beacon_aggregate_and_proof]: {type: GossipType.beacon_aggregate_and_proof}; [GossipType.beacon_attestation]: {type: GossipType.beacon_attestation; subnet: number}; [GossipType.voluntary_exit]: {type: GossipType.voluntary_exit}; @@ -71,7 +69,6 @@ export type SSZTypeOfGossipTopic = T extends {type: infer export type GossipTypeMap = { [GossipType.beacon_block]: allForks.SignedBeaconBlock; [GossipType.blob_sidecar]: deneb.SignedBlobSidecar; - [GossipType.beacon_block_and_blobs_sidecar]: deneb.SignedBeaconBlockAndBlobsSidecar; [GossipType.beacon_aggregate_and_proof]: phase0.SignedAggregateAndProof; [GossipType.beacon_attestation]: phase0.Attestation; [GossipType.voluntary_exit]: phase0.SignedVoluntaryExit; @@ -87,9 +84,6 @@ export type GossipTypeMap = { export type GossipFnByType = { [GossipType.beacon_block]: (signedBlock: allForks.SignedBeaconBlock) => Promise | void; [GossipType.blob_sidecar]: (signedBlobSidecar: deneb.SignedBlobSidecar) => Promise | void; - [GossipType.beacon_block_and_blobs_sidecar]: ( - signedBeaconBlockAndBlobsSidecar: deneb.SignedBeaconBlockAndBlobsSidecar - ) => Promise | void; [GossipType.beacon_aggregate_and_proof]: (aggregateAndProof: phase0.SignedAggregateAndProof) => Promise | void; [GossipType.beacon_attestation]: (attestation: phase0.Attestation) => Promise | void; [GossipType.voluntary_exit]: (voluntaryExit: phase0.SignedVoluntaryExit) => Promise | void; diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index 8c5a35e74cfd..de1a571c3330 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -6,6 +6,7 @@ import { ForkSeq, SYNC_COMMITTEE_SUBNET_COUNT, isForkLightClient, + MAX_BLOBS_PER_BLOCK, } from "@lodestar/params"; import {GossipAction, GossipActionError, GossipErrorCode} from "../../chain/errors/gossipValidation.js"; @@ -60,7 +61,6 @@ export function stringifyGossipTopic(forkDigestContext: ForkDigestContext, topic function stringifyGossipTopicType(topic: GossipTopic): string { switch (topic.type) { case GossipType.beacon_block: - case GossipType.beacon_block_and_blobs_sidecar: case GossipType.beacon_aggregate_and_proof: case GossipType.voluntary_exit: case GossipType.proposer_slashing: @@ -86,8 +86,6 @@ export function getGossipSSZType(topic: GossipTopic) { return ssz[topic.fork].SignedBeaconBlock; case GossipType.blob_sidecar: return ssz.deneb.SignedBlobSidecar; - case GossipType.beacon_block_and_blobs_sidecar: - return ssz.deneb.SignedBeaconBlockAndBlobsSidecar; case GossipType.beacon_aggregate_and_proof: return ssz.phase0.SignedAggregateAndProof; case GossipType.beacon_attestation: @@ -164,7 +162,6 @@ export function parseGossipTopic(forkDigestContext: ForkDigestContext, topicStr: // Inline-d the parseGossipTopicType() function since spreading the resulting object x4 the time to parse a topicStr switch (gossipTypeStr) { case GossipType.beacon_block: - case GossipType.beacon_block_and_blobs_sidecar: case GossipType.beacon_aggregate_and_proof: case GossipType.voluntary_exit: case GossipType.proposer_slashing: @@ -208,18 +205,18 @@ export function getCoreTopicsAtFork( ): GossipTopicTypeMap[keyof GossipTopicTypeMap][] { // Common topics for all forks const topics: GossipTopicTypeMap[keyof GossipTopicTypeMap][] = [ - // {type: GossipType.beacon_block}, // Handled below + {type: GossipType.beacon_block}, {type: GossipType.beacon_aggregate_and_proof}, {type: GossipType.voluntary_exit}, {type: GossipType.proposer_slashing}, {type: GossipType.attester_slashing}, ]; - // After Deneb only track beacon_block_and_blobs_sidecar topic - if (ForkSeq[fork] < ForkSeq.deneb) { - topics.push({type: GossipType.beacon_block}); - } else { - topics.push({type: GossipType.beacon_block_and_blobs_sidecar}); + // After Deneb also track blob_sidecar_{index} + if (ForkSeq[fork] >= ForkSeq.deneb) { + for (let index = 0; index < MAX_BLOBS_PER_BLOCK; index++) { + topics.push({type: GossipType.blob_sidecar, index}); + } } // capella @@ -265,7 +262,6 @@ function parseEncodingStr(encodingStr: string): GossipEncoding { export const gossipTopicIgnoreDuplicatePublishError: Record = { [GossipType.beacon_block]: true, [GossipType.blob_sidecar]: true, - [GossipType.beacon_block_and_blobs_sidecar]: true, [GossipType.beacon_aggregate_and_proof]: true, [GossipType.beacon_attestation]: true, [GossipType.voluntary_exit]: true, diff --git a/packages/beacon-node/src/network/processor/extractSlotRootFns.ts b/packages/beacon-node/src/network/processor/extractSlotRootFns.ts index 34b313e920b1..24fcfaae6cbc 100644 --- a/packages/beacon-node/src/network/processor/extractSlotRootFns.ts +++ b/packages/beacon-node/src/network/processor/extractSlotRootFns.ts @@ -4,7 +4,7 @@ import { getBlockRootFromSignedAggregateAndProofSerialized, getSlotFromAttestationSerialized, getSlotFromSignedAggregateAndProofSerialized, - getSlotFromSignedBeaconBlockAndBlobsSidecarSerialized, + getSlotFromSignedBlobSidecarSerialized, getSlotFromSignedBeaconBlockSerialized, } from "../../util/sszBytes.js"; import {GossipType} from "../gossip/index.js"; @@ -42,8 +42,8 @@ export function createExtractBlockSlotRootFns(): ExtractSlotRootFns { } return {slot}; }, - [GossipType.beacon_block_and_blobs_sidecar]: (data: Uint8Array): SlotOptionalRoot | null => { - const slot = getSlotFromSignedBeaconBlockAndBlobsSidecarSerialized(data); + [GossipType.blob_sidecar]: (data: Uint8Array): SlotOptionalRoot | null => { + const slot = getSlotFromSignedBlobSidecarSerialized(data); if (slot === null) { return null; diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index ed567b846209..455a4d34c1dc 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -35,7 +35,6 @@ import {NetworkEvent, NetworkEventBus} from "../events.js"; import {PeerAction} from "../peers/index.js"; import {validateLightClientFinalityUpdate} from "../../chain/validation/lightClientFinalityUpdate.js"; import {validateLightClientOptimisticUpdate} from "../../chain/validation/lightClientOptimisticUpdate.js"; -import {validateGossipBlobsSidecar} from "../../chain/validation/blobsSidecar.js"; import {BlockInput, BlockSource, getBlockInput} from "../../chain/blocks/types.js"; import {sszDeserialize} from "../gossip/topic.js"; import {INetworkCore} from "../core/index.js"; @@ -132,8 +131,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH ignoreIfKnown: true, // proposer signature already checked in validateBeaconBlock() validProposerSignature: true, - // blobsSidecar already checked in validateGossipBlobsSidecar() - validBlobsSidecar: true, + // blobSidecars already checked in validateGossipBlobSidecars() + validBlobSidecars: true, // It's critical to keep a good number of mesh peers. // To do that, the Gossip Job Wait Time should be consistently <3s to avoid the behavior penalties in gossip // Gossip Job Wait Time depends on the BLS Job Wait Time @@ -190,22 +189,6 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH // TODO DENEB: impl to be added on migration of blockinput }, - [GossipType.beacon_block_and_blobs_sidecar]: async ({serializedData}, topic, peerIdStr, seenTimestampSec) => { - const blockAndBlocks = sszDeserialize(topic, serializedData); - const {beaconBlock, blobsSidecar} = blockAndBlocks; - // TODO Deneb: Should throw for pre fork blocks? - if (config.getForkSeq(beaconBlock.message.slot) < ForkSeq.deneb) { - throw new GossipActionError(GossipAction.REJECT, {code: "PRE_DENEB_BLOCK"}); - } - - // Validate block + blob. Then forward, then handle both - // TODO DENEB: replace null with proper binary data for block and blobs separately - const blockInput = getBlockInput.postDeneb(config, beaconBlock, BlockSource.gossip, blobsSidecar, null); - await validateBeaconBlock(blockInput, topic.fork, peerIdStr, seenTimestampSec); - validateGossipBlobsSidecar(beaconBlock, blobsSidecar); - handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); - }, - [GossipType.beacon_aggregate_and_proof]: async ({serializedData}, topic, _peer, seenTimestampSec) => { let validationResult: AggregateAndProofValidationResult; const signedAggregateAndProof = sszDeserialize(topic, serializedData); diff --git a/packages/beacon-node/src/network/processor/gossipQueues.ts b/packages/beacon-node/src/network/processor/gossipQueues.ts index 63abd16ad3a8..389ba7acca81 100644 --- a/packages/beacon-node/src/network/processor/gossipQueues.ts +++ b/packages/beacon-node/src/network/processor/gossipQueues.ts @@ -41,12 +41,6 @@ const gossipQueueOpts: { type: QueueType.FIFO, dropOpts: {type: DropType.count, count: 1}, }, - // TODO DENEB: What's a good queue max given that now blocks are much bigger? - [GossipType.beacon_block_and_blobs_sidecar]: { - maxLength: 32, - type: QueueType.FIFO, - dropOpts: {type: DropType.count, count: 1}, - }, // lighthoue has aggregate_queue 4096 and unknown_block_aggregate_queue 1024, we use single queue [GossipType.beacon_aggregate_and_proof]: { maxLength: 5120, diff --git a/packages/beacon-node/src/network/processor/index.ts b/packages/beacon-node/src/network/processor/index.ts index 9d5d05e7a672..dbd7c56cf9df 100644 --- a/packages/beacon-node/src/network/processor/index.ts +++ b/packages/beacon-node/src/network/processor/index.ts @@ -57,7 +57,6 @@ type WorkOpts = { const executeGossipWorkOrderObj: Record = { [GossipType.beacon_block]: {bypassQueue: true}, [GossipType.blob_sidecar]: {bypassQueue: true}, - [GossipType.beacon_block_and_blobs_sidecar]: {bypassQueue: true}, [GossipType.beacon_aggregate_and_proof]: {}, [GossipType.voluntary_exit]: {}, [GossipType.bls_to_execution_change]: {}, @@ -239,7 +238,7 @@ export class NetworkProcessor { } if ( slot === this.chain.clock.currentSlot && - (topicType === GossipType.beacon_block || topicType === GossipType.beacon_block_and_blobs_sidecar) + (topicType === GossipType.beacon_block || topicType === GossipType.blob_sidecar) ) { // in the worse case if the current slot block is not valid, this will be reset in the next slot this.isProcessingCurrentSlotBlock = true; diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index 9604a5fb1ea4..46c28f7948fa 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -180,29 +180,28 @@ export function getSlotFromSignedBeaconBlockSerialized(data: Uint8Array): Slot | } /** - * 4 + 4 + SLOT_BYTES_POSITION_IN_SIGNED_BEACON_BLOCK = 4 + 4 + (4 + 96) = 108 - * class SignedBeaconBlockAndBlobsSidecar(Container): - * beaconBlock: SignedBeaconBlock [offset - 4 bytes] - * blobsSidecar: BlobsSidecar, + * 4 + 96 = 100 + * ``` + * class SignedBlobSidecar(Container): + * message: BlobSidecar [fixed] + * signature: BLSSignature [fixed] + * + * class BlobSidecar(Container): + * blockRoot: Root [fixed - 32 bytes ], + * index: BlobIndex [fixed - 8 bytes ], + * slot: Slot [fixed - 8 bytes] + * ... + * ``` */ -/** - * Variable size. - * class BlobsSidecar(Container): - * beaconBlockRoot: Root, - * beaconBlockSlot: Slot, - * blobs: Blobs, - * kzgAggregatedProof: KZGProof, - */ -const SLOT_BYTES_POSITION_IN_SIGNED_BEACON_BLOCK_AND_BLOBS_SIDECAR = - VARIABLE_FIELD_OFFSET + VARIABLE_FIELD_OFFSET + SLOT_BYTES_POSITION_IN_SIGNED_BEACON_BLOCK; +const SLOT_BYTES_POSITION_IN_SIGNED_BLOB_SIDECAR = 32 + 8; -export function getSlotFromSignedBeaconBlockAndBlobsSidecarSerialized(data: Uint8Array): Slot | null { - if (data.length < SLOT_BYTES_POSITION_IN_SIGNED_BEACON_BLOCK_AND_BLOBS_SIDECAR + SLOT_SIZE) { +export function getSlotFromSignedBlobSidecarSerialized(data: Uint8Array): Slot | null { + if (data.length < SLOT_BYTES_POSITION_IN_SIGNED_BLOB_SIDECAR + SLOT_SIZE) { return null; } - return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_SIGNED_BEACON_BLOCK_AND_BLOBS_SIDECAR); + return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_SIGNED_BLOB_SIDECAR); } function getSlotFromOffset(data: Uint8Array, offset: number): Slot { diff --git a/packages/beacon-node/test/e2e/network/network.test.ts b/packages/beacon-node/test/e2e/network/network.test.ts index 5126796074df..13cdd6c20d71 100644 --- a/packages/beacon-node/test/e2e/network/network.test.ts +++ b/packages/beacon-node/test/e2e/network/network.test.ts @@ -131,11 +131,11 @@ function runTests(this: Mocha.Suite, {useWorker}: {useWorker: boolean}): void { await netA.subscribeGossipCoreTopics(); expect(await getTopics(netA)).deep.equals([ + "/eth2/18ae4ccb/beacon_block/ssz_snappy", "/eth2/18ae4ccb/beacon_aggregate_and_proof/ssz_snappy", "/eth2/18ae4ccb/voluntary_exit/ssz_snappy", "/eth2/18ae4ccb/proposer_slashing/ssz_snappy", "/eth2/18ae4ccb/attester_slashing/ssz_snappy", - "/eth2/18ae4ccb/beacon_block/ssz_snappy", ]); await netA.unsubscribeGossipCoreTopics(); diff --git a/packages/beacon-node/test/spec/presets/fork_choice.ts b/packages/beacon-node/test/spec/presets/fork_choice.ts index 3ebe850c75e2..5fae3acc3a5e 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.ts @@ -169,7 +169,7 @@ export const forkChoiceTest = try { await chain.processBlock(blockImport, { seenTimestampSec: tickTime, - validBlobsSidecar: true, + validBlobSidecars: true, importAttestations: AttestationImportOpt.Force, }); if (!isValid) throw Error("Expect error since this is a negative test"); diff --git a/packages/beacon-node/test/unit/network/gossip/topic.test.ts b/packages/beacon-node/test/unit/network/gossip/topic.test.ts index c2e6bb5c8015..eee3f2fa3ff9 100644 --- a/packages/beacon-node/test/unit/network/gossip/topic.test.ts +++ b/packages/beacon-node/test/unit/network/gossip/topic.test.ts @@ -21,12 +21,6 @@ describe("network / gossip / topic", function () { topicStr: "/eth2/46acb19a/blob_sidecar_1/ssz_snappy", }, ], - [GossipType.beacon_block_and_blobs_sidecar]: [ - { - topic: {type: GossipType.beacon_block_and_blobs_sidecar, fork: ForkName.deneb, encoding}, - topicStr: "/eth2/46acb19a/beacon_block_and_blobs_sidecar/ssz_snappy", - }, - ], [GossipType.beacon_aggregate_and_proof]: [ { topic: {type: GossipType.beacon_aggregate_and_proof, fork: ForkName.phase0, encoding}, diff --git a/packages/beacon-node/test/unit/util/kzg.test.ts b/packages/beacon-node/test/unit/util/kzg.test.ts index 797136a17abf..6b6b92bd645e 100644 --- a/packages/beacon-node/test/unit/util/kzg.test.ts +++ b/packages/beacon-node/test/unit/util/kzg.test.ts @@ -4,6 +4,7 @@ import {BYTES_PER_FIELD_ELEMENT, BLOB_TX_TYPE} from "@lodestar/params"; import {kzgCommitmentToVersionedHash} from "@lodestar/state-transition"; import {loadEthereumTrustedSetup, initCKZG, ckzg, FIELD_ELEMENTS_PER_BLOB_MAINNET} from "../../../src/util/kzg.js"; +import {validateBlobSidecars, validateGossipBlobSidecar} from "../../../src/chain/validation/blobSidecar.js"; import {getMockBeaconChain} from "../../utils/mocks/chain.js"; describe("C-KZG", async () => { @@ -68,7 +69,12 @@ describe("C-KZG", async () => { expect(signedBlobSidecars.length).to.equal(2); - // TODO DENEB: add full validation + // Full validation + validateBlobSidecars(slot, blockRoot, kzgCommitments, blobSidecars); + + signedBlobSidecars.forEach(async (signedBlobSidecar) => { + await validateGossipBlobSidecar(chain.config, chain, signedBlobSidecar, signedBlobSidecar.message.index); + }); }); }); diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index a3d9f54aab54..58b39dda82bf 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -11,7 +11,7 @@ import { getSlotFromSignedAggregateAndProofSerialized, getSignatureFromAttestationSerialized, getSlotFromSignedBeaconBlockSerialized, - getSlotFromSignedBeaconBlockAndBlobsSidecarSerialized, + getSlotFromSignedBlobSidecarSerialized, } from "../../../src/util/sszBytes.js"; describe("attestation SSZ serialized picking", () => { @@ -148,25 +148,20 @@ describe("signedBeaconBlock SSZ serialized picking", () => { }); }); -describe("signedBeaconBlockAndBlobsSidecar SSZ serialized picking", () => { - const testCases = [ - ssz.deneb.SignedBeaconBlockAndBlobsSidecar.defaultValue(), - signedBeaconBlockAndBlobsSidecarFromValues(1_000_000), - ]; +describe("signedBlobSidecar SSZ serialized picking", () => { + const testCases = [ssz.deneb.SignedBlobSidecar.defaultValue(), signedBlobSidecarFromValues(1_000_000)]; - for (const [i, signedBeaconBlockAndBlobsSidecar] of testCases.entries()) { - const bytes = ssz.deneb.SignedBeaconBlockAndBlobsSidecar.serialize(signedBeaconBlockAndBlobsSidecar); - it(`signedBeaconBlockAndBlobsSidecar ${i}`, () => { - expect(getSlotFromSignedBeaconBlockAndBlobsSidecarSerialized(bytes)).equals( - signedBeaconBlockAndBlobsSidecar.beaconBlock.message.slot - ); + for (const [i, signedBlobSidecar] of testCases.entries()) { + const bytes = ssz.deneb.SignedBlobSidecar.serialize(signedBlobSidecar); + it(`signedBlobSidecar ${i}`, () => { + expect(getSlotFromSignedBlobSidecarSerialized(bytes)).equals(signedBlobSidecar.message.slot); }); } - it("getSlotFromSignedBeaconBlockAndBlobsSidecarSerialized - invalid data", () => { - const invalidSlotDataSizes = [0, 50, 112]; + it("signedBlobSidecar - invalid data", () => { + const invalidSlotDataSizes = [0, 20, 38]; for (const size of invalidSlotDataSizes) { - expect(getSlotFromSignedBeaconBlockAndBlobsSidecarSerialized(Buffer.alloc(size))).to.be.null; + expect(getSlotFromSignedBlobSidecarSerialized(Buffer.alloc(size))).to.be.null; } }); }); @@ -205,8 +200,8 @@ function signedBeaconBlockFromValues(slot: Slot): phase0.SignedBeaconBlock { return signedBeaconBlock; } -function signedBeaconBlockAndBlobsSidecarFromValues(slot: Slot): deneb.SignedBeaconBlockAndBlobsSidecar { - const signedBeaconBlockAndBlobsSidecar = ssz.deneb.SignedBeaconBlockAndBlobsSidecar.defaultValue(); - signedBeaconBlockAndBlobsSidecar.beaconBlock.message.slot = slot; - return signedBeaconBlockAndBlobsSidecar; +function signedBlobSidecarFromValues(slot: Slot): deneb.SignedBlobSidecar { + const signedBlobSidecar = ssz.deneb.SignedBlobSidecar.defaultValue(); + signedBlobSidecar.message.slot = slot; + return signedBlobSidecar; } diff --git a/packages/beacon-node/test/utils/node/beacon.ts b/packages/beacon-node/test/utils/node/beacon.ts index bb9c4546d77d..6c62e22939d3 100644 --- a/packages/beacon-node/test/utils/node/beacon.ts +++ b/packages/beacon-node/test/utils/node/beacon.ts @@ -80,9 +80,9 @@ export async function getDevBeaconNode( await db.blockArchive.add(block); if (config.getForkSeq(GENESIS_SLOT) >= ForkSeq.deneb) { - const blobsSidecar = ssz.deneb.BlobsSidecar.defaultValue(); - blobsSidecar.beaconBlockRoot = config.getForkTypes(GENESIS_SLOT).BeaconBlock.hashTreeRoot(block.message); - await db.blobsSidecar.add(blobsSidecar); + const blobSidecars = ssz.deneb.BlobSidecars.defaultValue(); + const blockRoot = config.getForkTypes(GENESIS_SLOT).BeaconBlock.hashTreeRoot(block.message); + await db.blobSidecars.add({blobSidecars, slot: GENESIS_SLOT, blockRoot}); } } diff --git a/packages/types/src/deneb/sszTypes.ts b/packages/types/src/deneb/sszTypes.ts index 70101a0135ac..eb04bb1e7a38 100644 --- a/packages/types/src/deneb/sszTypes.ts +++ b/packages/types/src/deneb/sszTypes.ts @@ -54,6 +54,7 @@ export const BlindedBlob = Bytes32; export const BlindedBlobs = new ListCompositeType(BlindedBlob, MAX_BLOBS_PER_BLOCK); export const VersionedHash = Bytes32; export const BlobKzgCommitments = new ListCompositeType(KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK); +export const KZGProofs = new ListCompositeType(KZGProof, MAX_BLOBS_PER_BLOCK); // Constants diff --git a/packages/types/src/deneb/types.ts b/packages/types/src/deneb/types.ts index a4d2286af1e6..f91e94a68190 100644 --- a/packages/types/src/deneb/types.ts +++ b/packages/types/src/deneb/types.ts @@ -18,6 +18,7 @@ export type SignedBlindedBlobSidecar = ValueOf; export type BlobKzgCommitments = ValueOf; +export type KZGProofs = ValueOf; export type Polynomial = ValueOf; export type PolynomialAndCommitment = ValueOf; export type BLSFieldElement = ValueOf; From 9ba6d543311c9e086451124860034c0b92a25b4d Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 27 Jun 2023 17:47:02 +0200 Subject: [PATCH 35/96] chore: fix yarn install warnings (#5711) --- package.json | 4 ++++ packages/beacon-node/package.json | 1 + packages/reqresp/package.json | 3 ++- yarn.lock | 28 ++++++++++++++++++++++++---- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 4e7497462b35..991dbb393c0c 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "karma-spec-reporter": "^0.0.36", "karma-webpack": "^5.0.0", "lerna": "^6.6.1", + "libp2p": "0.42.2", "mocha": "^10.2.0", "node-gyp": "^9.3.1", "npm-run-all": "^4.1.5", @@ -85,5 +86,8 @@ "typescript": "^5.0.3", "typescript-docs-verifier": "^2.4.0", "webpack": "^5.77.0" + }, + "resolutions": { + "dns-over-http-resolver": "^2.1.1" } } diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 39c04a7f9f5f..94182c1a71d0 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -97,6 +97,7 @@ "@chainsafe/as-chacha20poly1305": "^0.1.0", "@chainsafe/as-sha256": "^0.3.1", "@chainsafe/bls": "7.1.1", + "@chainsafe/blst": "^0.2.9", "@chainsafe/discv5": "^3.0.0", "@chainsafe/libp2p-gossipsub": "^6.2.0", "@chainsafe/libp2p-noise": "^11.0.4", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 3e0c037757bf..0444a6c93381 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -68,7 +68,8 @@ }, "devDependencies": { "@lodestar/logger": "^1.9.0", - "@lodestar/types": "^1.9.0" + "@lodestar/types": "^1.9.0", + "libp2p": "0.42.2" }, "peerDependencies": { "libp2p": "~0.42.2" diff --git a/yarn.lock b/yarn.lock index ddbce881cbbe..33a596cc5298 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5266,6 +5266,13 @@ builtins@^5.0.0: dependencies: semver "^7.0.0" +busboy@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893" + integrity sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA== + dependencies: + streamsearch "^1.1.0" + byline@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/byline/-/byline-5.0.0.tgz#741c5216468eadc457b03410118ad77de8c1ddb1" @@ -6563,14 +6570,15 @@ dir-glob@^3.0.1: dependencies: path-type "^4.0.0" -dns-over-http-resolver@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/dns-over-http-resolver/-/dns-over-http-resolver-2.1.0.tgz#e3f13182b46b60e0be2473f3fbfc4ec5bbfb9539" - integrity sha512-eb8RGy6k54JdD7Rjw8g65y1MyA4z3m3IIYh7uazkgZuKIdFn8gYt8dydMm3op+2UshDdk9EexrXcDluKNY/CDg== +dns-over-http-resolver@^2.1.0, dns-over-http-resolver@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/dns-over-http-resolver/-/dns-over-http-resolver-2.1.1.tgz#a3ff3fd7614cea7a4b72594eaf12fb3c85080456" + integrity sha512-Lm/eXB7yAQLJ5WxlBGwYfBY7utduXPZykcSmcG6K7ozM0wrZFvxZavhT6PqI0kd/5CUTfev/RrEFQqyU4CGPew== dependencies: debug "^4.3.1" native-fetch "^4.0.2" receptacle "^1.3.2" + undici "^5.12.0" dns-packet@^4.0.0: version "4.2.0" @@ -14134,6 +14142,11 @@ streamroller@^3.1.1: debug "^4.3.4" fs-extra "^8.1.0" +streamsearch@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764" + integrity sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg== + strict-event-emitter-types@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/strict-event-emitter-types/-/strict-event-emitter-types-2.0.0.tgz" @@ -15035,6 +15048,13 @@ unbox-primitive@^1.0.0, unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" +undici@^5.12.0: + version "5.22.1" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.22.1.tgz#877d512effef2ac8be65e695f3586922e1a57d7b" + integrity sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw== + dependencies: + busboy "^1.6.0" + unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230" From e35fae02e7310edf43a0f63cee6756dd6745e681 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 27 Jun 2023 17:51:48 +0200 Subject: [PATCH 36/96] fix: explicitly exit process after beacon node closed (#5716) --- packages/cli/src/cmds/beacon/handler.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cli/src/cmds/beacon/handler.ts b/packages/cli/src/cmds/beacon/handler.ts index af34cf861697..4c71e4714dd2 100644 --- a/packages/cli/src/cmds/beacon/handler.ts +++ b/packages/cli/src/cmds/beacon/handler.ts @@ -130,6 +130,9 @@ export async function beaconHandler(args: BeaconArgs & GlobalArgs): Promise Date: Tue, 27 Jun 2023 19:45:59 +0300 Subject: [PATCH 37/96] ci: assert yarn prints no warnings (#5565) * Assert in CI yarn prints no warnings * Add exit 1 --- .github/workflows/test.yml | 3 +++ scripts/assert_no_yarn_warnings.sh | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100755 scripts/assert_no_yarn_warnings.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6a73ac6841e1..b95fb6a44794 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,9 @@ jobs: path: packages/validator/spec-tests key: spec-test-data-${{ hashFiles('packages/validator/test/spec/params.ts') }} + - name: Assert yarn prints no warnings + run: scripts/assert_no_yarn_warnings.sh + # Misc sanity checks - name: Lint Grafana dashboards run: scripts/validate-grafana-dashboards.sh diff --git a/scripts/assert_no_yarn_warnings.sh b/scripts/assert_no_yarn_warnings.sh new file mode 100755 index 000000000000..60dace730f92 --- /dev/null +++ b/scripts/assert_no_yarn_warnings.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# run yarn install --check-files, capturing stderr +OUTPUT=$(yarn install --check-files 2>&1) + +echo $OUTPUT + +# grep the output for 'warning' +if echo "$OUTPUT" | grep -qi 'warning'; then + echo "There were warnings in yarn install --check-files" + exit 1 +else + echo "No warnings in yarn install --check-files" +fi From cc7802f9486e9b1fb6d0cbef6d7f6e14cf7f9c26 Mon Sep 17 00:00:00 2001 From: g11tech Date: Wed, 28 Jun 2023 12:27:46 +0530 Subject: [PATCH 38/96] feat: freeze voluntary exit signature fork domain to Capella version from deneb fork onwards (#5688) * feat: freeze voluntary exit signature fork domain to Deneb * fix the domain version to capella * skip deneb signed voluntary exits in spec * also skip random as spec tests as they test exits across deneb boundary * add getDomainForVoluntaryExit * fix domain fork --- packages/beacon-node/test/spec/presets/index.test.ts | 5 +++++ packages/cli/src/cmds/validator/voluntaryExit.ts | 3 +-- packages/config/src/genesisConfig/index.ts | 12 +++++++++++- packages/config/src/genesisConfig/types.ts | 2 ++ .../src/signatureSets/voluntaryExits.ts | 3 +-- packages/validator/src/services/validatorStore.ts | 3 +-- 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/packages/beacon-node/test/spec/presets/index.test.ts b/packages/beacon-node/test/spec/presets/index.test.ts index c45f859ec1e5..1e42ddde4725 100644 --- a/packages/beacon-node/test/spec/presets/index.test.ts +++ b/packages/beacon-node/test/spec/presets/index.test.ts @@ -37,6 +37,11 @@ const skipOpts: SkipOpts = { skippedPrefixes: [ "capella/light_client/single_merkle_proof/BeaconBlockBody", "deneb/light_client/single_merkle_proof/BeaconBlockBody", + // TODO: deneb + // Deneb signed voluntary exits will not be valid so skipping this + // To be cleaned up with the spec version update + "deneb/operations/voluntary_exit/", + "deneb/random/random", ], }; diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index eda73b6e6d99..666003604576 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -5,7 +5,6 @@ import { computeStartSlotAtEpoch, getCurrentSlot, } from "@lodestar/state-transition"; -import {DOMAIN_VOLUNTARY_EXIT} from "@lodestar/params"; import {createBeaconConfig} from "@lodestar/config"; import {ssz, phase0} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; @@ -97,7 +96,7 @@ ${validatorsToExit.map((v) => `${v.pubkey} ${v.index} ${v.status}`).join("\n")}` } for (const [i, {index, signer, pubkey}] of validatorsToExit.entries()) { - const domain = config.getDomain(computeStartSlotAtEpoch(exitEpoch), DOMAIN_VOLUNTARY_EXIT); + const domain = config.getDomainForVoluntaryExit(computeStartSlotAtEpoch(exitEpoch)); const voluntaryExit: phase0.VoluntaryExit = {epoch: exitEpoch, validatorIndex: index}; const signingRoot = computeSigningRoot(ssz.phase0.VoluntaryExit, voluntaryExit, domain); diff --git a/packages/config/src/genesisConfig/index.ts b/packages/config/src/genesisConfig/index.ts index 6035e192e1fc..48d7ee8bf32f 100644 --- a/packages/config/src/genesisConfig/index.ts +++ b/packages/config/src/genesisConfig/index.ts @@ -1,4 +1,4 @@ -import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {ForkName, SLOTS_PER_EPOCH, DOMAIN_VOLUNTARY_EXIT} from "@lodestar/params"; import {DomainType, ForkDigest, phase0, Root, Slot, ssz, Version} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "../beaconConfig.js"; @@ -76,6 +76,16 @@ export function createCachedGenesis(chainForkConfig: ChainForkConfig, genesisVal return domain; }, + getDomainForVoluntaryExit(stateSlot: Slot, messageSlot?: Slot) { + // Deneb onwards the signature domain fork is fixed to capella + const domain = + stateSlot < chainForkConfig.DENEB_FORK_EPOCH * SLOTS_PER_EPOCH + ? this.getDomain(stateSlot, DOMAIN_VOLUNTARY_EXIT, messageSlot) + : this.getDomainAtFork(ForkName.capella, DOMAIN_VOLUNTARY_EXIT); + + return domain; + }, + forkDigest2ForkName(forkDigest: ForkDigest | ForkDigestHex): ForkName { const forkDigestHex = toHexStringNoPrefix(forkDigest); const forkName = forkNameByForkDigest.get(forkDigestHex); diff --git a/packages/config/src/genesisConfig/types.ts b/packages/config/src/genesisConfig/types.ts index bd961af02063..7bd1ba57eea7 100644 --- a/packages/config/src/genesisConfig/types.ts +++ b/packages/config/src/genesisConfig/types.ts @@ -22,5 +22,7 @@ export interface CachedGenesis extends ForkDigestContext { */ getDomainAtFork(forkName: ForkName, domainType: DomainType): Uint8Array; + getDomainForVoluntaryExit(stateSlot: Slot, messageSlot?: Slot): Uint8Array; + readonly genesisValidatorsRoot: Root; } diff --git a/packages/state-transition/src/signatureSets/voluntaryExits.ts b/packages/state-transition/src/signatureSets/voluntaryExits.ts index b3c6e6f28473..bb86ef41777a 100644 --- a/packages/state-transition/src/signatureSets/voluntaryExits.ts +++ b/packages/state-transition/src/signatureSets/voluntaryExits.ts @@ -1,4 +1,3 @@ -import {DOMAIN_VOLUNTARY_EXIT} from "@lodestar/params"; import {allForks, phase0, ssz} from "@lodestar/types"; import { computeSigningRoot, @@ -25,7 +24,7 @@ export function getVoluntaryExitSignatureSet( ): ISignatureSet { const {epochCtx} = state; const slot = computeStartSlotAtEpoch(signedVoluntaryExit.message.epoch); - const domain = state.config.getDomain(state.slot, DOMAIN_VOLUNTARY_EXIT, slot); + const domain = state.config.getDomainForVoluntaryExit(state.slot, slot); return { type: SignatureSetType.single, diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index d2fe36b1e214..4482eed8727a 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -17,7 +17,6 @@ import { DOMAIN_SELECTION_PROOF, DOMAIN_SYNC_COMMITTEE, DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, - DOMAIN_VOLUNTARY_EXIT, DOMAIN_APPLICATION_BUILDER, DOMAIN_BLOB_SIDECAR, } from "@lodestar/params"; @@ -563,7 +562,7 @@ export class ValidatorStore { exitEpoch: Epoch ): Promise { const signingSlot = computeStartSlotAtEpoch(exitEpoch); - const domain = this.config.getDomain(signingSlot, DOMAIN_VOLUNTARY_EXIT); + const domain = this.config.getDomainForVoluntaryExit(signingSlot); const voluntaryExit: phase0.VoluntaryExit = {epoch: exitEpoch, validatorIndex}; const signingRoot = computeSigningRoot(ssz.phase0.VoluntaryExit, voluntaryExit, domain); From 9a0a031a7023b702c7b4077ddea43448847762ed Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 28 Jun 2023 23:35:50 +0300 Subject: [PATCH 39/96] Schedule gnosis shapella (#5717) --- packages/config/src/chainConfig/networks/gnosis.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/config/src/chainConfig/networks/gnosis.ts b/packages/config/src/chainConfig/networks/gnosis.ts index 56e15ce0aedf..c952352f8438 100644 --- a/packages/config/src/chainConfig/networks/gnosis.ts +++ b/packages/config/src/chainConfig/networks/gnosis.ts @@ -40,5 +40,5 @@ export const gnosisChainConfig: ChainConfig = { BELLATRIX_FORK_EPOCH: 385536, // Capella CAPELLA_FORK_VERSION: b("0x03000064"), - CAPELLA_FORK_EPOCH: Infinity, + CAPELLA_FORK_EPOCH: 648704, // 2023-08-01T11:34:20.000Z }; From a4c93c13258f01ed078609c4333739d047b43a5d Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 29 Jun 2023 16:20:26 +0200 Subject: [PATCH 40/96] chore: fix local testnet dev scripts (#5719) --- scripts/dev/node1.sh | 7 ++----- scripts/dev/node2.sh | 10 +++++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/scripts/dev/node1.sh b/scripts/dev/node1.sh index b7a3d86caa2f..4260458b4d97 100755 --- a/scripts/dev/node1.sh +++ b/scripts/dev/node1.sh @@ -2,7 +2,7 @@ GENESIS_TIME=$(date +%s) -packages/cli/bin/lodestar dev \ +./lodestar dev \ --genesisValidators 8 \ --startValidators 0..7 \ --genesisTime $GENESIS_TIME \ @@ -13,7 +13,4 @@ packages/cli/bin/lodestar dev \ --rest.namespace '*' \ --metrics \ --logLevel debug \ - --eth1 false \ - --network.requestCountPeerLimit 1000000 \ - --network.blockCountTotalLimit 1000000 \ - --network.blockCountPeerLimit 1000000 + --eth1 false diff --git a/scripts/dev/node2.sh b/scripts/dev/node2.sh index a8a87416b23e..4906887c15e6 100755 --- a/scripts/dev/node2.sh +++ b/scripts/dev/node2.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Fetch node1 data -ENR=$(curl -s http://localhost:9596/eth/v1/node/identity | jq .data.enr) -GENESIS_TIME=$(curl -s http://localhost:9596/eth/v1/beacon/genesis | jq .data.genesis_time) +ENR=$(curl -s http://localhost:9596/eth/v1/node/identity | jq -r .data.enr) +GENESIS_TIME=$(curl -s http://localhost:9596/eth/v1/beacon/genesis | jq -r .data.genesis_time) -packages/cli/bin/lodestar dev \ +./lodestar dev \ --genesisValidators 8 \ --genesisTime $GENESIS_TIME \ --enr.ip 127.0.0.1 \ @@ -13,10 +13,10 @@ packages/cli/bin/lodestar dev \ --rest \ --rest.namespace '*' \ --metrics \ - --metrics.serverPort 8009 \ + --metrics.port 8009 \ --logLevel debug \ --eth1 false \ --port 9001 \ --rest.port 9597 \ --network.connectToDiscv5Bootnodes true \ - --network.discv5.bootEnrs $ENR \ No newline at end of file + --bootnodes $ENR From 7028ffbcb897e054ffc2158a99d0a46725eef99f Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 30 Jun 2023 11:17:09 +0200 Subject: [PATCH 41/96] deps: update typescript and related dependencies (#5720) Update typescript and dependencies --- package.json | 16 +- yarn.lock | 569 ++++++++++++++++++++++++++------------------------- 2 files changed, 298 insertions(+), 287 deletions(-) diff --git a/package.json b/package.json index 991dbb393c0c..2fe8168b3e1d 100644 --- a/package.json +++ b/package.json @@ -44,14 +44,14 @@ "@types/node": "^18.15.11", "@types/sinon": "^10.0.13", "@types/sinon-chai": "^3.2.9", - "@typescript-eslint/eslint-plugin": "5.57.1", - "@typescript-eslint/parser": "5.57.1", + "@typescript-eslint/eslint-plugin": "5.60.1", + "@typescript-eslint/parser": "5.60.1", "chai": "^4.3.7", "chai-as-promised": "^7.1.1", "codecov": "^3.8.3", "crypto-browserify": "^3.12.0", "electron": "^21.0.1", - "eslint": "^8.37.0", + "eslint": "^8.43.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-chai-expect": "^3.0.0", @@ -73,7 +73,7 @@ "npm-run-all": "^4.1.5", "nyc": "^15.1.0", "path-browserify": "^1.0.1", - "prettier": "^2.8.7", + "prettier": "^2.8.8", "process": "^0.11.10", "resolve-typescript-plugin": "^2.0.1", "sinon": "^15.0.3", @@ -81,11 +81,11 @@ "stream-browserify": "^3.0.0", "stream-http": "^3.2.0", "supertest": "^6.3.3", - "ts-loader": "^9.4.2", + "ts-loader": "^9.4.4", "ts-node": "^10.9.1", - "typescript": "^5.0.3", - "typescript-docs-verifier": "^2.4.0", - "webpack": "^5.77.0" + "typescript": "^5.1.6", + "typescript-docs-verifier": "^2.5.0", + "webpack": "^5.88.1" }, "resolutions": { "dns-over-http-resolver": "^2.1.1" diff --git a/yarn.lock b/yarn.lock index 33a596cc5298..dba94ac93595 100644 --- a/yarn.lock +++ b/yarn.lock @@ -722,14 +722,14 @@ resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.5.0.tgz#f6f729b02feee2c749f57e334b7a1b5f40a81724" integrity sha512-vITaYzIcNmjn5tF5uxcZ/ft7/RXGrMUIS9HalWckEOF6ESiwXKoMzAQf2UW0aVd6rnOeExTJVd5hmWXucBKGXQ== -"@eslint/eslintrc@^2.0.2": - version "2.0.2" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.0.2.tgz#01575e38707add677cf73ca1589abba8da899a02" - integrity sha512-3W4f5tDUra+pA+FzgugqL2pRimUTDJWKr7BINqOpkZrC0uYI0NIc0/JFgBROCU07HR6GieA5m3/rsPIhDmCXTQ== +"@eslint/eslintrc@^2.0.3": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.0.3.tgz#4910db5505f4d503f27774bf356e3704818a0331" + integrity sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ== dependencies: ajv "^6.12.4" debug "^4.3.2" - espree "^9.5.1" + espree "^9.5.2" globals "^13.19.0" ignore "^5.2.0" import-fresh "^3.2.1" @@ -737,10 +737,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.37.0": - version "8.37.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.37.0.tgz#cf1b5fa24217fe007f6487a26d765274925efa7d" - integrity sha512-x5vzdtOOGgFVDCUs81QRB2+liax8rFg3+7hqM+QhBG0/G3F1ZsoYl97UrqgHgQ9KKT7G6c4V+aTUCgu/n22v1A== +"@eslint/js@8.43.0": + version "8.43.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.43.0.tgz#559ca3d9ddbd6bf907ad524320a0d14b85586af0" + integrity sha512-s2UHCoiXfxMvmfzqoN+vrQ84ahUSYde9qNO1MdxmoEhyHWsfmwOpFlwYV+ePJEVc7gFnATGUi376WowX1N7tFg== "@ethereumjs/block@^4.2.2": version "4.2.2" @@ -1625,10 +1625,10 @@ resolved "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz" integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== -"@humanwhocodes/config-array@^0.11.8": - version "0.11.8" - resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.8.tgz#03595ac2075a4dc0f191cc2131de14fbd7d410b9" - integrity sha512-UybHIJzJnR5Qc/MsD9Kr+RpO2h+/P1GhOwdiLPXK5TWk5sgTdu88bTD9UP+CKbPPh5Rni1u0GjAdYQLemG8g+g== +"@humanwhocodes/config-array@^0.11.10": + version "0.11.10" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.10.tgz#5a3ffe32cc9306365fb3fd572596cd602d5e12d2" + integrity sha512-KVVjQmNUepDVGXNuoRRdmmEjruj0KfiGSbS8LVc12LMsWDQzRXJ0qdhN8L8uUigKpfEHRhlaQFY0ib1tnUbNeQ== dependencies: "@humanwhocodes/object-schema" "^1.2.1" debug "^4.1.1" @@ -1677,34 +1677,17 @@ dependencies: "@sinclair/typebox" "^0.25.16" -"@jridgewell/gen-mapping@^0.3.0": - version "0.3.2" - resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9" - integrity sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A== - dependencies: - "@jridgewell/set-array" "^1.0.1" - "@jridgewell/sourcemap-codec" "^1.4.10" - "@jridgewell/trace-mapping" "^0.3.9" - -"@jridgewell/resolve-uri@^3.0.3": +"@jridgewell/resolve-uri@3.1.0", "@jridgewell/resolve-uri@^3.0.3": version "3.1.0" resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== -"@jridgewell/set-array@^1.0.1": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" - integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== - -"@jridgewell/source-map@^0.3.2": - version "0.3.2" - resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.2.tgz#f45351aaed4527a298512ec72f81040c998580fb" - integrity sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw== - dependencies: - "@jridgewell/gen-mapping" "^0.3.0" - "@jridgewell/trace-mapping" "^0.3.9" +"@jridgewell/source-map@^0.3.3": + version "0.3.4" + resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.4.tgz#856a142864530d4059dda415659b48d37db2d556" + integrity sha512-KE/SxsDqNs3rrWwFHcRh15ZLVFrI0YoZtgAdIyIq9k5hUNmiWRXXThPomIxHuL20sLdgzbDFyvkUMna14bvtrw== -"@jridgewell/sourcemap-codec@^1.4.10": +"@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10": version "1.4.14" resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== @@ -1717,13 +1700,13 @@ "@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/sourcemap-codec" "^1.4.10" -"@jridgewell/trace-mapping@^0.3.7", "@jridgewell/trace-mapping@^0.3.9": - version "0.3.14" - resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.14.tgz#b231a081d8f66796e475ad588a1ef473112701ed" - integrity sha512-bJWEfQ9lPTvm3SneWwRFVLzrh6nhjwqw7TUFFBEMzwvg7t7PCDenf2lDwqo4NQXzdpgBXyFgDWnQA+2vkruksQ== +"@jridgewell/trace-mapping@^0.3.17": + version "0.3.18" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz#25783b2086daf6ff1dcb53c9249ae480e4dd4cd6" + integrity sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA== dependencies: - "@jridgewell/resolve-uri" "^3.0.3" - "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/resolve-uri" "3.1.0" + "@jridgewell/sourcemap-codec" "1.4.14" "@lerna/child-process@6.6.1": version "6.6.1" @@ -3571,10 +3554,10 @@ resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.0.tgz#5fb2e536c1ae9bf35366eed879e827fa59ca41c2" integrity sha512-WulqXMDUTYAXCjZnk6JtIHPigp55cVtDgDrO2gHRwhyJto21+1zbVCtOYB2L1F9w4qCQ0rOGWBnBe0FNTiEJIQ== -"@types/estree@^0.0.51": - version "0.0.51" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.51.tgz#cfd70924a25a3fd32b218e5e420e6897e1ac4f40" - integrity sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ== +"@types/estree@^1.0.0": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.1.tgz#aa22750962f3bf0e79d753d3cc067f010c95f194" + integrity sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA== "@types/eventsource@^1.1.11": version "1.1.11" @@ -3958,15 +3941,15 @@ dependencies: "@types/node" "*" -"@typescript-eslint/eslint-plugin@5.57.1": - version "5.57.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.57.1.tgz#d1ab162a3cd2671b8a1c9ddf6e2db73b14439735" - integrity sha512-1MeobQkQ9tztuleT3v72XmY0XuKXVXusAhryoLuU5YZ+mXoYKZP9SQ7Flulh1NX4DTjpGTc2b/eMu4u7M7dhnQ== +"@typescript-eslint/eslint-plugin@5.60.1": + version "5.60.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.60.1.tgz#81382d6ecb92b8dda70e91f9035611cb2fecd1c3" + integrity sha512-KSWsVvsJsLJv3c4e73y/Bzt7OpqMCADUO846bHcuWYSYM19bldbAeDv7dYyV0jwkbMfJ2XdlzwjhXtuD7OY6bw== dependencies: "@eslint-community/regexpp" "^4.4.0" - "@typescript-eslint/scope-manager" "5.57.1" - "@typescript-eslint/type-utils" "5.57.1" - "@typescript-eslint/utils" "5.57.1" + "@typescript-eslint/scope-manager" "5.60.1" + "@typescript-eslint/type-utils" "5.60.1" + "@typescript-eslint/utils" "5.60.1" debug "^4.3.4" grapheme-splitter "^1.0.4" ignore "^5.2.0" @@ -3974,193 +3957,193 @@ semver "^7.3.7" tsutils "^3.21.0" -"@typescript-eslint/parser@5.57.1": - version "5.57.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-5.57.1.tgz#af911234bd4401d09668c5faf708a0570a17a748" - integrity sha512-hlA0BLeVSA/wBPKdPGxoVr9Pp6GutGoY380FEhbVi0Ph4WNe8kLvqIRx76RSQt1lynZKfrXKs0/XeEk4zZycuA== +"@typescript-eslint/parser@5.60.1": + version "5.60.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-5.60.1.tgz#0f2f58209c0862a73e3d5a56099abfdfa21d0fd3" + integrity sha512-pHWlc3alg2oSMGwsU/Is8hbm3XFbcrb6P5wIxcQW9NsYBfnrubl/GhVVD/Jm/t8HXhA2WncoIRfBtnCgRGV96Q== dependencies: - "@typescript-eslint/scope-manager" "5.57.1" - "@typescript-eslint/types" "5.57.1" - "@typescript-eslint/typescript-estree" "5.57.1" + "@typescript-eslint/scope-manager" "5.60.1" + "@typescript-eslint/types" "5.60.1" + "@typescript-eslint/typescript-estree" "5.60.1" debug "^4.3.4" -"@typescript-eslint/scope-manager@5.57.1": - version "5.57.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.57.1.tgz#5d28799c0fc8b501a29ba1749d827800ef22d710" - integrity sha512-N/RrBwEUKMIYxSKl0oDK5sFVHd6VI7p9K5MyUlVYAY6dyNb/wHUqndkTd3XhpGlXgnQsBkRZuu4f9kAHghvgPw== +"@typescript-eslint/scope-manager@5.60.1": + version "5.60.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.60.1.tgz#35abdb47f500c68c08f2f2b4f22c7c79472854bb" + integrity sha512-Dn/LnN7fEoRD+KspEOV0xDMynEmR3iSHdgNsarlXNLGGtcUok8L4N71dxUgt3YvlO8si7E+BJ5Fe3wb5yUw7DQ== dependencies: - "@typescript-eslint/types" "5.57.1" - "@typescript-eslint/visitor-keys" "5.57.1" + "@typescript-eslint/types" "5.60.1" + "@typescript-eslint/visitor-keys" "5.60.1" -"@typescript-eslint/type-utils@5.57.1": - version "5.57.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-5.57.1.tgz#235daba621d3f882b8488040597b33777c74bbe9" - integrity sha512-/RIPQyx60Pt6ga86hKXesXkJ2WOS4UemFrmmq/7eOyiYjYv/MUSHPlkhU6k9T9W1ytnTJueqASW+wOmW4KrViw== +"@typescript-eslint/type-utils@5.60.1": + version "5.60.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-5.60.1.tgz#17770540e98d65ab4730c7aac618003f702893f4" + integrity sha512-vN6UztYqIu05nu7JqwQGzQKUJctzs3/Hg7E2Yx8rz9J+4LgtIDFWjjl1gm3pycH0P3mHAcEUBd23LVgfrsTR8A== dependencies: - "@typescript-eslint/typescript-estree" "5.57.1" - "@typescript-eslint/utils" "5.57.1" + "@typescript-eslint/typescript-estree" "5.60.1" + "@typescript-eslint/utils" "5.60.1" debug "^4.3.4" tsutils "^3.21.0" -"@typescript-eslint/types@5.57.1": - version "5.57.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.57.1.tgz#d9989c7a9025897ea6f0550b7036027f69e8a603" - integrity sha512-bSs4LOgyV3bJ08F5RDqO2KXqg3WAdwHCu06zOqcQ6vqbTJizyBhuh1o1ImC69X4bV2g1OJxbH71PJqiO7Y1RuA== +"@typescript-eslint/types@5.60.1": + version "5.60.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.60.1.tgz#a17473910f6b8d388ea83c9d7051af89c4eb7561" + integrity sha512-zDcDx5fccU8BA0IDZc71bAtYIcG9PowaOwaD8rjYbqwK7dpe/UMQl3inJ4UtUK42nOCT41jTSCwg76E62JpMcg== -"@typescript-eslint/typescript-estree@5.57.1": - version "5.57.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.57.1.tgz#10d9643e503afc1ca4f5553d9bbe672ea4050b71" - integrity sha512-A2MZqD8gNT0qHKbk2wRspg7cHbCDCk2tcqt6ScCFLr5Ru8cn+TCfM786DjPhqwseiS+PrYwcXht5ztpEQ6TFTw== +"@typescript-eslint/typescript-estree@5.60.1": + version "5.60.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.60.1.tgz#8c71824b7165b64d5ebd7aa42968899525959834" + integrity sha512-hkX70J9+2M2ZT6fhti5Q2FoU9zb+GeZK2SLP1WZlvUDqdMbEKhexZODD1WodNRyO8eS+4nScvT0dts8IdaBzfw== dependencies: - "@typescript-eslint/types" "5.57.1" - "@typescript-eslint/visitor-keys" "5.57.1" + "@typescript-eslint/types" "5.60.1" + "@typescript-eslint/visitor-keys" "5.60.1" debug "^4.3.4" globby "^11.1.0" is-glob "^4.0.3" semver "^7.3.7" tsutils "^3.21.0" -"@typescript-eslint/utils@5.57.1": - version "5.57.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.57.1.tgz#0f97b0bbd88c2d5e2036869f26466be5f4c69475" - integrity sha512-kN6vzzf9NkEtawECqze6v99LtmDiUJCVpvieTFA1uL7/jDghiJGubGZ5csicYHU1Xoqb3oH/R5cN5df6W41Nfg== +"@typescript-eslint/utils@5.60.1": + version "5.60.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.60.1.tgz#6861ebedbefba1ac85482d2bdef6f2ff1eb65b80" + integrity sha512-tiJ7FFdFQOWssFa3gqb94Ilexyw0JVxj6vBzaSpfN/8IhoKkDuSAenUKvsSHw2A/TMpJb26izIszTXaqygkvpQ== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@types/json-schema" "^7.0.9" "@types/semver" "^7.3.12" - "@typescript-eslint/scope-manager" "5.57.1" - "@typescript-eslint/types" "5.57.1" - "@typescript-eslint/typescript-estree" "5.57.1" + "@typescript-eslint/scope-manager" "5.60.1" + "@typescript-eslint/types" "5.60.1" + "@typescript-eslint/typescript-estree" "5.60.1" eslint-scope "^5.1.1" semver "^7.3.7" -"@typescript-eslint/visitor-keys@5.57.1": - version "5.57.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.57.1.tgz#585e5fa42a9bbcd9065f334fd7c8a4ddfa7d905e" - integrity sha512-RjQrAniDU0CEk5r7iphkm731zKlFiUjvcBS2yHAg8WWqFMCaCrD0rKEVOMUyMMcbGPZ0bPp56srkGWrgfZqLRA== +"@typescript-eslint/visitor-keys@5.60.1": + version "5.60.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.60.1.tgz#19a877358bf96318ec35d90bfe6bd1445cce9434" + integrity sha512-xEYIxKcultP6E/RMKqube11pGjXH1DCo60mQoWhVYyKfLkwbIVVjYxmOenNMxILx0TjCujPTjjnTIVzm09TXIw== dependencies: - "@typescript-eslint/types" "5.57.1" + "@typescript-eslint/types" "5.60.1" eslint-visitor-keys "^3.3.0" -"@webassemblyjs/ast@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.1.tgz#2bfd767eae1a6996f432ff7e8d7fc75679c0b6a7" - integrity sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw== +"@webassemblyjs/ast@1.11.6", "@webassemblyjs/ast@^1.11.5": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.6.tgz#db046555d3c413f8966ca50a95176a0e2c642e24" + integrity sha512-IN1xI7PwOvLPgjcf180gC1bqn3q/QaOCwYUahIOhbYUu8KA/3tw2RT/T0Gidi1l7Hhj5D/INhJxiICObqpMu4Q== dependencies: - "@webassemblyjs/helper-numbers" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" + "@webassemblyjs/helper-numbers" "1.11.6" + "@webassemblyjs/helper-wasm-bytecode" "1.11.6" -"@webassemblyjs/floating-point-hex-parser@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz#f6c61a705f0fd7a6aecaa4e8198f23d9dc179e4f" - integrity sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ== +"@webassemblyjs/floating-point-hex-parser@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz#dacbcb95aff135c8260f77fa3b4c5fea600a6431" + integrity sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw== -"@webassemblyjs/helper-api-error@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz#1a63192d8788e5c012800ba6a7a46c705288fd16" - integrity sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg== +"@webassemblyjs/helper-api-error@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz#6132f68c4acd59dcd141c44b18cbebbd9f2fa768" + integrity sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q== -"@webassemblyjs/helper-buffer@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz#832a900eb444884cde9a7cad467f81500f5e5ab5" - integrity sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA== +"@webassemblyjs/helper-buffer@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.6.tgz#b66d73c43e296fd5e88006f18524feb0f2c7c093" + integrity sha512-z3nFzdcp1mb8nEOFFk8DrYLpHvhKC3grJD2ardfKOzmbmJvEf/tPIqCY+sNcwZIY8ZD7IkB2l7/pqhUhqm7hLA== -"@webassemblyjs/helper-numbers@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz#64d81da219fbbba1e3bd1bfc74f6e8c4e10a62ae" - integrity sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ== +"@webassemblyjs/helper-numbers@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz#cbce5e7e0c1bd32cf4905ae444ef64cea919f1b5" + integrity sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g== dependencies: - "@webassemblyjs/floating-point-hex-parser" "1.11.1" - "@webassemblyjs/helper-api-error" "1.11.1" + "@webassemblyjs/floating-point-hex-parser" "1.11.6" + "@webassemblyjs/helper-api-error" "1.11.6" "@xtuc/long" "4.2.2" -"@webassemblyjs/helper-wasm-bytecode@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz#f328241e41e7b199d0b20c18e88429c4433295e1" - integrity sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q== +"@webassemblyjs/helper-wasm-bytecode@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz#bb2ebdb3b83aa26d9baad4c46d4315283acd51e9" + integrity sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA== -"@webassemblyjs/helper-wasm-section@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz#21ee065a7b635f319e738f0dd73bfbda281c097a" - integrity sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg== +"@webassemblyjs/helper-wasm-section@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.6.tgz#ff97f3863c55ee7f580fd5c41a381e9def4aa577" + integrity sha512-LPpZbSOwTpEC2cgn4hTydySy1Ke+XEu+ETXuoyvuyezHO3Kjdu90KK95Sh9xTbmjrCsUwvWwCOQQNta37VrS9g== dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-buffer" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/wasm-gen" "1.11.1" + "@webassemblyjs/ast" "1.11.6" + "@webassemblyjs/helper-buffer" "1.11.6" + "@webassemblyjs/helper-wasm-bytecode" "1.11.6" + "@webassemblyjs/wasm-gen" "1.11.6" -"@webassemblyjs/ieee754@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz#963929e9bbd05709e7e12243a099180812992614" - integrity sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ== +"@webassemblyjs/ieee754@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz#bb665c91d0b14fffceb0e38298c329af043c6e3a" + integrity sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg== dependencies: "@xtuc/ieee754" "^1.2.0" -"@webassemblyjs/leb128@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/leb128/-/leb128-1.11.1.tgz#ce814b45574e93d76bae1fb2644ab9cdd9527aa5" - integrity sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw== +"@webassemblyjs/leb128@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/leb128/-/leb128-1.11.6.tgz#70e60e5e82f9ac81118bc25381a0b283893240d7" + integrity sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ== dependencies: "@xtuc/long" "4.2.2" -"@webassemblyjs/utf8@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/utf8/-/utf8-1.11.1.tgz#d1f8b764369e7c6e6bae350e854dec9a59f0a3ff" - integrity sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ== - -"@webassemblyjs/wasm-edit@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz#ad206ebf4bf95a058ce9880a8c092c5dec8193d6" - integrity sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-buffer" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/helper-wasm-section" "1.11.1" - "@webassemblyjs/wasm-gen" "1.11.1" - "@webassemblyjs/wasm-opt" "1.11.1" - "@webassemblyjs/wasm-parser" "1.11.1" - "@webassemblyjs/wast-printer" "1.11.1" - -"@webassemblyjs/wasm-gen@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz#86c5ea304849759b7d88c47a32f4f039ae3c8f76" - integrity sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/ieee754" "1.11.1" - "@webassemblyjs/leb128" "1.11.1" - "@webassemblyjs/utf8" "1.11.1" - -"@webassemblyjs/wasm-opt@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz#657b4c2202f4cf3b345f8a4c6461c8c2418985f2" - integrity sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-buffer" "1.11.1" - "@webassemblyjs/wasm-gen" "1.11.1" - "@webassemblyjs/wasm-parser" "1.11.1" - -"@webassemblyjs/wasm-parser@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz#86ca734534f417e9bd3c67c7a1c75d8be41fb199" - integrity sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-api-error" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/ieee754" "1.11.1" - "@webassemblyjs/leb128" "1.11.1" - "@webassemblyjs/utf8" "1.11.1" - -"@webassemblyjs/wast-printer@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz#d0c73beda8eec5426f10ae8ef55cee5e7084c2f0" - integrity sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg== - dependencies: - "@webassemblyjs/ast" "1.11.1" +"@webassemblyjs/utf8@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/utf8/-/utf8-1.11.6.tgz#90f8bc34c561595fe156603be7253cdbcd0fab5a" + integrity sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA== + +"@webassemblyjs/wasm-edit@^1.11.5": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.6.tgz#c72fa8220524c9b416249f3d94c2958dfe70ceab" + integrity sha512-Ybn2I6fnfIGuCR+Faaz7YcvtBKxvoLV3Lebn1tM4o/IAJzmi9AWYIPWpyBfU8cC+JxAO57bk4+zdsTjJR+VTOw== + dependencies: + "@webassemblyjs/ast" "1.11.6" + "@webassemblyjs/helper-buffer" "1.11.6" + "@webassemblyjs/helper-wasm-bytecode" "1.11.6" + "@webassemblyjs/helper-wasm-section" "1.11.6" + "@webassemblyjs/wasm-gen" "1.11.6" + "@webassemblyjs/wasm-opt" "1.11.6" + "@webassemblyjs/wasm-parser" "1.11.6" + "@webassemblyjs/wast-printer" "1.11.6" + +"@webassemblyjs/wasm-gen@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.6.tgz#fb5283e0e8b4551cc4e9c3c0d7184a65faf7c268" + integrity sha512-3XOqkZP/y6B4F0PBAXvI1/bky7GryoogUtfwExeP/v7Nzwo1QLcq5oQmpKlftZLbT+ERUOAZVQjuNVak6UXjPA== + dependencies: + "@webassemblyjs/ast" "1.11.6" + "@webassemblyjs/helper-wasm-bytecode" "1.11.6" + "@webassemblyjs/ieee754" "1.11.6" + "@webassemblyjs/leb128" "1.11.6" + "@webassemblyjs/utf8" "1.11.6" + +"@webassemblyjs/wasm-opt@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.6.tgz#d9a22d651248422ca498b09aa3232a81041487c2" + integrity sha512-cOrKuLRE7PCe6AsOVl7WasYf3wbSo4CeOk6PkrjS7g57MFfVUF9u6ysQBBODX0LdgSvQqRiGz3CXvIDKcPNy4g== + dependencies: + "@webassemblyjs/ast" "1.11.6" + "@webassemblyjs/helper-buffer" "1.11.6" + "@webassemblyjs/wasm-gen" "1.11.6" + "@webassemblyjs/wasm-parser" "1.11.6" + +"@webassemblyjs/wasm-parser@1.11.6", "@webassemblyjs/wasm-parser@^1.11.5": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.6.tgz#bb85378c527df824004812bbdb784eea539174a1" + integrity sha512-6ZwPeGzMJM3Dqp3hCsLgESxBGtT/OeCvCZ4TA1JUPYgmhAx38tTPR9JaKy0S5H3evQpO/h2uWs2j6Yc/fjkpTQ== + dependencies: + "@webassemblyjs/ast" "1.11.6" + "@webassemblyjs/helper-api-error" "1.11.6" + "@webassemblyjs/helper-wasm-bytecode" "1.11.6" + "@webassemblyjs/ieee754" "1.11.6" + "@webassemblyjs/leb128" "1.11.6" + "@webassemblyjs/utf8" "1.11.6" + +"@webassemblyjs/wast-printer@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.11.6.tgz#a7bf8dd7e362aeb1668ff43f35cb849f188eff20" + integrity sha512-JM7AhRcE+yW2GWYaKeHL5vt4xqee5N2WcezptmgyhNS+ScggqcT1OtXykhAb13Sn5Yas0j2uv9tHgrjwvzAP4A== + dependencies: + "@webassemblyjs/ast" "1.11.6" "@xtuc/long" "4.2.2" "@xtuc/ieee754@^1.2.0": @@ -4269,10 +4252,10 @@ accepts@~1.3.4, accepts@~1.3.8: mime-types "~2.1.34" negotiator "0.6.3" -acorn-import-assertions@^1.7.6: - version "1.8.0" - resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9" - integrity sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw== +acorn-import-assertions@^1.9.0: + version "1.9.0" + resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz#507276249d684797c84e0734ef84860334cfb1ac" + integrity sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA== acorn-jsx@^5.3.2: version "5.3.2" @@ -4289,16 +4272,16 @@ acorn@^8.4.1, acorn@^8.7.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== -acorn@^8.5.0: - version "8.8.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.0.tgz#88c0187620435c7f6015803f5539dae05a9dbea8" - integrity sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w== - acorn@^8.8.0: version "8.8.2" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== +acorn@^8.8.2: + version "8.9.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.9.0.tgz#78a16e3b2bcc198c10822786fa6679e245db5b59" + integrity sha512-jaVNAFBHNLXspO543WnNNPZFRtavh3skAkITqD0/2aeMkKZTN+254PyhwxFYrk3vQ1xfY+2wbesJMs/JC8/PwQ== + add-stream@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/add-stream/-/add-stream-1.0.0.tgz#6a7990437ca736d5e1288db92bd3266d5f5cb2aa" @@ -6784,7 +6767,7 @@ engine.io@~6.2.0: engine.io-parser "~5.0.3" ws "~8.2.3" -enhanced-resolve@^5.0.0, enhanced-resolve@^5.10.0: +enhanced-resolve@^5.0.0: version "5.10.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz#0dc579c3bb2a1032e357ac45b8f3a6f3ad4fb1e6" integrity sha512-T0yTFjdpldGY8PmuXXR0PyQ1ufZpEGiHVrp7zHKB7jdR4qlmZHhONVM5AQOAWXuF/w3dnHbEQVrNptJgt7F+cQ== @@ -6792,6 +6775,14 @@ enhanced-resolve@^5.0.0, enhanced-resolve@^5.10.0: graceful-fs "^4.2.4" tapable "^2.2.0" +enhanced-resolve@^5.15.0: + version "5.15.0" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz#1af946c7d93603eb88e9896cee4904dc012e9c35" + integrity sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg== + dependencies: + graceful-fs "^4.2.4" + tapable "^2.2.0" + enquirer@~2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" @@ -6952,10 +6943,10 @@ es-abstract@^1.20.4: unbox-primitive "^1.0.2" which-typed-array "^1.1.9" -es-module-lexer@^0.9.0: - version "0.9.3" - resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-0.9.3.tgz#6f13db00cc38417137daf74366f535c8eb438f19" - integrity sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ== +es-module-lexer@^1.2.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-1.3.0.tgz#6be9c9e0b4543a60cd166ff6f8b4e9dae0b0c16f" + integrity sha512-vZK7T0N2CBmBOixhmjdqx2gWVbFZ4DXZ/NyRMZVlJXPa7CyFS+/a4QQsDGDQy9ZfEzxFuNEsMLeQJnKP2p5/JA== es-set-tostringtag@^2.0.1: version "2.0.1" @@ -7121,10 +7112,10 @@ eslint-scope@5.1.1, eslint-scope@^5.1.1: esrecurse "^4.3.0" estraverse "^4.1.1" -eslint-scope@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.1.1.tgz#fff34894c2f65e5226d3041ac480b4513a163642" - integrity sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw== +eslint-scope@^7.2.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.2.0.tgz#f21ebdafda02352f103634b96dd47d9f81ca117b" + integrity sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw== dependencies: esrecurse "^4.3.0" estraverse "^5.2.0" @@ -7158,21 +7149,21 @@ eslint-visitor-keys@^3.3.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== -eslint-visitor-keys@^3.4.0: - version "3.4.0" - resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz#c7f0f956124ce677047ddbc192a68f999454dedc" - integrity sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ== +eslint-visitor-keys@^3.4.1: + version "3.4.1" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz#c22c48f48942d08ca824cc526211ae400478a994" + integrity sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA== -eslint@^8.37.0: - version "8.37.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.37.0.tgz#1f660ef2ce49a0bfdec0b0d698e0b8b627287412" - integrity sha512-NU3Ps9nI05GUoVMxcZx1J8CNR6xOvUT4jAUMH5+z8lpp3aEdPVCImKw6PWG4PY+Vfkpr+jvMpxs/qoE7wq0sPw== +eslint@^8.43.0: + version "8.43.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.43.0.tgz#3e8c6066a57097adfd9d390b8fc93075f257a094" + integrity sha512-aaCpf2JqqKesMFGgmRPessmVKjcGXqdlAYLLC3THM8t5nBRZRQ+st5WM/hoJXkdioEXLLbXgclUpM0TXo5HX5Q== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.4.0" - "@eslint/eslintrc" "^2.0.2" - "@eslint/js" "8.37.0" - "@humanwhocodes/config-array" "^0.11.8" + "@eslint/eslintrc" "^2.0.3" + "@eslint/js" "8.43.0" + "@humanwhocodes/config-array" "^0.11.10" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" ajv "^6.10.0" @@ -7181,9 +7172,9 @@ eslint@^8.37.0: debug "^4.3.2" doctrine "^3.0.0" escape-string-regexp "^4.0.0" - eslint-scope "^7.1.1" - eslint-visitor-keys "^3.4.0" - espree "^9.5.1" + eslint-scope "^7.2.0" + eslint-visitor-keys "^3.4.1" + espree "^9.5.2" esquery "^1.4.2" esutils "^2.0.2" fast-deep-equal "^3.1.3" @@ -7191,13 +7182,12 @@ eslint@^8.37.0: find-up "^5.0.0" glob-parent "^6.0.2" globals "^13.19.0" - grapheme-splitter "^1.0.4" + graphemer "^1.4.0" ignore "^5.2.0" import-fresh "^3.0.0" imurmurhash "^0.1.4" is-glob "^4.0.0" is-path-inside "^3.0.3" - js-sdsl "^4.1.4" js-yaml "^4.1.0" json-stable-stringify-without-jsonify "^1.0.1" levn "^0.4.1" @@ -7214,14 +7204,14 @@ esm@^3.2.25: resolved "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz" integrity sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA== -espree@^9.5.1: - version "9.5.1" - resolved "https://registry.yarnpkg.com/espree/-/espree-9.5.1.tgz#4f26a4d5f18905bf4f2e0bd99002aab807e96dd4" - integrity sha512-5yxtHSZXRSW5pvv3hAlXM5+/Oswi1AUFqBmbibKb5s6bp3rGIDkyXU6xCoyuuLhijr4SFwPrXRoZjz0AZDN9tg== +espree@^9.5.2: + version "9.5.2" + resolved "https://registry.yarnpkg.com/espree/-/espree-9.5.2.tgz#e994e7dc33a082a7a82dceaf12883a829353215b" + integrity sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw== dependencies: acorn "^8.8.0" acorn-jsx "^5.3.2" - eslint-visitor-keys "^3.4.0" + eslint-visitor-keys "^3.4.1" esprima@^4.0.0: version "4.0.1" @@ -8601,6 +8591,11 @@ grapheme-splitter@^1.0.4: resolved "https://registry.yarnpkg.com/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz#9cf3a665c6247479896834af35cf1dbb4400767e" integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ== +graphemer@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" + integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== + handlebars@^4.7.7: version "4.7.7" resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1" @@ -12656,10 +12651,10 @@ prettier-linter-helpers@^1.0.0: dependencies: fast-diff "^1.1.2" -prettier@^2.8.7: - version "2.8.7" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.7.tgz#bb79fc8729308549d28fe3a98fce73d2c0656450" - integrity sha512-yPngTo3aXUUmyuTjeTUT75txrf+aMh9FiD7q9ZE/i6r0bPb22g4FsE6Y338PQX1bmfy08i9QQCB7/rcUAVntfw== +prettier@^2.8.8: + version "2.8.8" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.8.tgz#e8c5d7e98a4305ffe3de2e1fc4aca1a71c28b1da" + integrity sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q== pretty-format@29.4.3: version "29.4.3" @@ -13567,7 +13562,7 @@ sax@>=0.6.0, sax@^1.2.4: resolved "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== -schema-utils@^3.1.0, schema-utils@^3.1.1: +schema-utils@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.1.1.tgz#bc74c4b6b6995c1d88f76a8b77bea7219e0c8281" integrity sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw== @@ -13576,6 +13571,15 @@ schema-utils@^3.1.0, schema-utils@^3.1.1: ajv "^6.12.5" ajv-keywords "^3.5.2" +schema-utils@^3.2.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.3.0.tgz#f50a88877c3c01652a15b622ae9e9795df7a60fe" + integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== + dependencies: + "@types/json-schema" "^7.0.8" + ajv "^6.12.5" + ajv-keywords "^3.5.2" + scrypt-js@3.0.1, scrypt-js@^3.0.0, scrypt-js@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/scrypt-js/-/scrypt-js-3.0.1.tgz" @@ -13657,13 +13661,20 @@ serialize-error@^7.0.1: dependencies: type-fest "^0.13.1" -serialize-javascript@6.0.0, serialize-javascript@^6.0.0: +serialize-javascript@6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz" integrity sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag== dependencies: randombytes "^2.1.0" +serialize-javascript@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.1.tgz#b206efb27c3da0b0ab6b52f48d170b7996458e5c" + integrity sha512-owoXEFjWRllis8/M1Q+Cw5k8ZH40e3zhp/ovX+Xr/vi1qj6QesbyXXViFbpNvWvPNAD62SutwEXavefrLJWj7w== + dependencies: + randombytes "^2.1.0" + serve-static@1.15.0: version "1.15.0" resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" @@ -14517,24 +14528,24 @@ tempy@1.0.0: type-fest "^0.16.0" unique-string "^2.0.0" -terser-webpack-plugin@^5.1.3: - version "5.3.3" - resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.3.tgz#8033db876dd5875487213e87c627bca323e5ed90" - integrity sha512-Fx60G5HNYknNTNQnzQ1VePRuu89ZVYWfjRAeT5rITuCY/1b08s49e5kSQwHDirKZWuoKOBRFS98EUUoZ9kLEwQ== +terser-webpack-plugin@^5.3.7: + version "5.3.9" + resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.9.tgz#832536999c51b46d468067f9e37662a3b96adfe1" + integrity sha512-ZuXsqE07EcggTWQjXUj+Aot/OMcD0bMKGgF63f7UxYcu5/AJF53aIpK1YoP5xR9l6s/Hy2b+t1AM0bLNPRuhwA== dependencies: - "@jridgewell/trace-mapping" "^0.3.7" + "@jridgewell/trace-mapping" "^0.3.17" jest-worker "^27.4.5" schema-utils "^3.1.1" - serialize-javascript "^6.0.0" - terser "^5.7.2" + serialize-javascript "^6.0.1" + terser "^5.16.8" -terser@^5.7.2: - version "5.14.2" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.14.2.tgz#9ac9f22b06994d736174f4091aa368db896f1c10" - integrity sha512-oL0rGeM/WFQCUd0y2QrWxYnq7tfSuKBiqTjRPWrRgB46WD/kiwHwF8T23z78H6Q6kGCuuHcPB+KULHRdxvVGQA== +terser@^5.16.8: + version "5.18.2" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.18.2.tgz#ff3072a0faf21ffd38f99acc9a0ddf7b5f07b948" + integrity sha512-Ah19JS86ypbJzTzvUCX7KOsEIhDaRONungA4aYBjEP3JZRf4ocuDzTg4QWZnPn9DEMiMYGJPiSOy7aykoCc70w== dependencies: - "@jridgewell/source-map" "^0.3.2" - acorn "^8.5.0" + "@jridgewell/source-map" "^0.3.3" + acorn "^8.8.2" commander "^2.20.0" source-map-support "~0.5.20" @@ -14739,10 +14750,10 @@ truncate-utf8-bytes@^1.0.0: dependencies: utf8-byte-length "^1.0.1" -ts-loader@^9.4.2: - version "9.4.2" - resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-9.4.2.tgz#80a45eee92dd5170b900b3d00abcfa14949aeb78" - integrity sha512-OmlC4WVmFv5I0PpaxYb+qGeGOdm5giHU7HwDDUjw59emP2UYMHy9fFSDcYgSNoH8sXcj4hGCSEhlDZ9ULeDraA== +ts-loader@^9.4.4: + version "9.4.4" + resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-9.4.4.tgz#6ceaf4d58dcc6979f84125335904920884b7cee4" + integrity sha512-MLukxDHBl8OJ5Dk3y69IsKVFRA/6MwzEqBgh+OXMPB/OD01KQuWPFd1WAQP8a5PeSCAxfnkhiuWqfmFJzJQt9w== dependencies: chalk "^4.1.0" enhanced-resolve "^5.0.0" @@ -14960,10 +14971,10 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA== -typescript-docs-verifier@^2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/typescript-docs-verifier/-/typescript-docs-verifier-2.4.0.tgz#2eede027b54e385b790656ee61b8f68673b3d6ad" - integrity sha512-1beOPxmO7M2XSRT06SRgotUmYZ21OTYeQfnS1nnAIxWxbJymwCGRMBLSDnIOIO+7g4AoxnCiSLahQ80EpqFyig== +typescript-docs-verifier@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/typescript-docs-verifier/-/typescript-docs-verifier-2.5.0.tgz#5a39c89b492aca31100b20affd477913daa82f7d" + integrity sha512-h+1fW9LEJi5Q8fMZxdpVoXjGQohx7CAYTylF5JWmmj6IM0J21HgII1vpLAX/Q5B+jlIg8V7v7sGfwBI7LIG4oA== dependencies: chalk "^4.1.2" fs-extra "^10.0.0" @@ -14978,10 +14989,10 @@ typescript-docs-verifier@^2.4.0: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.8.3.tgz#d59344522c4bc464a65a730ac695007fdb66dd88" integrity sha512-goMHfm00nWPa8UvR/CPSvykqf6dVV8x/dp0c5mFTMTIu0u0FlGWRioyy7Nn0PGAdHxpJZnuO/ut+PpQ8UiHAig== -typescript@^5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.0.3.tgz#fe976f0c826a88d0a382007681cbb2da44afdedf" - integrity sha512-xv8mOEDnigb/tN9PSMTwSEqAnUvkoXMQlicOb0IUVDBSQCgBSaAAROUZYy2IcUy5qU6XajK5jjjO7TMWqBTKZA== +typescript@^5.1.6: + version "5.1.6" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.1.6.tgz#02f8ac202b6dad2c0dd5e0913745b47a37998274" + integrity sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA== ua-parser-js@^0.7.30: version "0.7.33" @@ -15610,22 +15621,22 @@ webpack-sources@^3.2.3: resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.3.tgz#2d4daab8451fd4b240cc27055ff6a0c2ccea0cde" integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== -webpack@^5.77.0: - version "5.77.0" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.77.0.tgz#dea3ad16d7ea6b84aa55fa42f4eac9f30e7eb9b4" - integrity sha512-sbGNjBr5Ya5ss91yzjeJTLKyfiwo5C628AFjEa6WSXcZa4E+F57om3Cc8xLb1Jh0b243AWuSYRf3dn7HVeFQ9Q== +webpack@^5.88.1: + version "5.88.1" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.88.1.tgz#21eba01e81bd5edff1968aea726e2fbfd557d3f8" + integrity sha512-FROX3TxQnC/ox4N+3xQoWZzvGXSuscxR32rbzjpXgEzWudJFEJBpdlkkob2ylrv5yzzufD1zph1OoFsLtm6stQ== dependencies: "@types/eslint-scope" "^3.7.3" - "@types/estree" "^0.0.51" - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/wasm-edit" "1.11.1" - "@webassemblyjs/wasm-parser" "1.11.1" + "@types/estree" "^1.0.0" + "@webassemblyjs/ast" "^1.11.5" + "@webassemblyjs/wasm-edit" "^1.11.5" + "@webassemblyjs/wasm-parser" "^1.11.5" acorn "^8.7.1" - acorn-import-assertions "^1.7.6" + acorn-import-assertions "^1.9.0" browserslist "^4.14.5" chrome-trace-event "^1.0.2" - enhanced-resolve "^5.10.0" - es-module-lexer "^0.9.0" + enhanced-resolve "^5.15.0" + es-module-lexer "^1.2.1" eslint-scope "5.1.1" events "^3.2.0" glob-to-regexp "^0.4.1" @@ -15634,9 +15645,9 @@ webpack@^5.77.0: loader-runner "^4.2.0" mime-types "^2.1.27" neo-async "^2.6.2" - schema-utils "^3.1.0" + schema-utils "^3.2.0" tapable "^2.1.1" - terser-webpack-plugin "^5.1.3" + terser-webpack-plugin "^5.3.7" watchpack "^2.4.0" webpack-sources "^3.2.3" From 9ea69efa0940b58e8c52cbe93c93c111200b56e8 Mon Sep 17 00:00:00 2001 From: g11tech Date: Fri, 30 Jun 2023 14:58:16 +0530 Subject: [PATCH 42/96] refactor: remove now defunct network zhejiang (#5721) refac: remove now defunct network zhejiang --- packages/cli/src/networks/index.ts | 6 +-- packages/cli/src/networks/zhejiang.ts | 10 ----- .../src/chainConfig/networks/zhejiang.ts | 41 ------------------- packages/config/src/networks.ts | 9 +--- 4 files changed, 2 insertions(+), 64 deletions(-) delete mode 100644 packages/cli/src/networks/zhejiang.ts delete mode 100644 packages/config/src/chainConfig/networks/zhejiang.ts diff --git a/packages/cli/src/networks/index.ts b/packages/cli/src/networks/index.ts index fb4b8dbe72ee..049cca9b002d 100644 --- a/packages/cli/src/networks/index.ts +++ b/packages/cli/src/networks/index.ts @@ -16,9 +16,8 @@ import * as goerli from "./goerli.js"; import * as ropsten from "./ropsten.js"; import * as sepolia from "./sepolia.js"; import * as chiado from "./chiado.js"; -import * as zhejiang from "./zhejiang.js"; -export type NetworkName = "mainnet" | "dev" | "gnosis" | "goerli" | "ropsten" | "sepolia" | "chiado" | "zhejiang"; +export type NetworkName = "mainnet" | "dev" | "gnosis" | "goerli" | "ropsten" | "sepolia" | "chiado"; export const networkNames: NetworkName[] = [ "mainnet", "gnosis", @@ -26,7 +25,6 @@ export const networkNames: NetworkName[] = [ "ropsten", "sepolia", "chiado", - "zhejiang", // Leave always as last network. The order matters for the --help printout "dev", @@ -66,8 +64,6 @@ export function getNetworkData(network: NetworkName): { return sepolia; case "chiado": return chiado; - case "zhejiang": - return zhejiang; default: throw Error(`Network not supported: ${network}`); } diff --git a/packages/cli/src/networks/zhejiang.ts b/packages/cli/src/networks/zhejiang.ts deleted file mode 100644 index 4b4d42516a92..000000000000 --- a/packages/cli/src/networks/zhejiang.ts +++ /dev/null @@ -1,10 +0,0 @@ -export {zhejiangChainConfig as chainConfig} from "@lodestar/config/networks"; - -export const depositContractDeployBlock = 0; -export const genesisFileUrl = - "https://raw.githubusercontent.com/ethpandaops/withdrawals-testnet/master/zhejiang-testnet/custom_config_data/genesis.ssz"; -export const bootnodesFileUrl = - "https://raw.githubusercontent.com/ethpandaops/withdrawals-testnet/master/zhejiang-testnet/custom_config_data/bootstrap_nodes.txt"; - -// Pick from above file -export const bootEnrs = []; diff --git a/packages/config/src/chainConfig/networks/zhejiang.ts b/packages/config/src/chainConfig/networks/zhejiang.ts deleted file mode 100644 index 42bab23110ab..000000000000 --- a/packages/config/src/chainConfig/networks/zhejiang.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString as b} from "@chainsafe/ssz"; -import {ChainConfig} from "../types.js"; -import {chainConfig as mainnet} from "../presets/mainnet.js"; - -// Zhejiang beacon chain config: -// https://github.com/eth-clients/merge-testnets/blob/main/sepolia-beacon-chain/config.yaml - -export const zhejiangChainConfig: ChainConfig = { - ...mainnet, - - CONFIG_NAME: "zhejiang", - - // Genesis - // --------------------------------------------------------------- - MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 58000, - MIN_GENESIS_TIME: 1675263480, - GENESIS_FORK_VERSION: b("0x00000069"), - GENESIS_DELAY: 120, - - // Forking - // --------------------------------------------------------------- - // # Altair - ALTAIR_FORK_VERSION: b("0x00000070"), - ALTAIR_FORK_EPOCH: 0, - // # Merge - BELLATRIX_FORK_VERSION: b("0x00000071"), - BELLATRIX_FORK_EPOCH: 0, - TERMINAL_TOTAL_DIFFICULTY: BigInt("0"), - // Capella - CAPELLA_FORK_VERSION: b("0x00000072"), - CAPELLA_FORK_EPOCH: 1350, - // Deneb - DENEB_FORK_VERSION: b("0x00000073"), - - // Deposit contract - // --------------------------------------------------------------- - DEPOSIT_CHAIN_ID: 1337803, - DEPOSIT_NETWORK_ID: 1337803, - DEPOSIT_CONTRACT_ADDRESS: b("0x4242424242424242424242424242424242424242"), -}; diff --git a/packages/config/src/networks.ts b/packages/config/src/networks.ts index 956f26b4f7ef..3f3c8da3cabe 100644 --- a/packages/config/src/networks.ts +++ b/packages/config/src/networks.ts @@ -5,7 +5,6 @@ import {goerliChainConfig} from "./chainConfig/networks/goerli.js"; import {ropstenChainConfig} from "./chainConfig/networks/ropsten.js"; import {sepoliaChainConfig} from "./chainConfig/networks/sepolia.js"; import {chiadoChainConfig} from "./chainConfig/networks/chiado.js"; -import {zhejiangChainConfig} from "./chainConfig/networks/zhejiang.js"; export { mainnetChainConfig, @@ -14,10 +13,9 @@ export { ropstenChainConfig, sepoliaChainConfig, chiadoChainConfig, - zhejiangChainConfig, }; -export type NetworkName = "mainnet" | "gnosis" | "goerli" | "ropsten" | "sepolia" | "chiado" | "zhejiang"; +export type NetworkName = "mainnet" | "gnosis" | "goerli" | "ropsten" | "sepolia" | "chiado"; export const networksChainConfig: Record = { mainnet: mainnetChainConfig, gnosis: gnosisChainConfig, @@ -25,7 +23,6 @@ export const networksChainConfig: Record = { ropsten: ropstenChainConfig, sepolia: sepoliaChainConfig, chiado: chiadoChainConfig, - zhejiang: zhejiangChainConfig, }; export type GenesisData = { @@ -58,8 +55,4 @@ export const genesisData: Record = { genesisTime: 1665396300, genesisValidatorsRoot: "0x9d642dac73058fbf39c0ae41ab1e34e4d889043cb199851ded7095bc99eb4c1e", }, - zhejiang: { - genesisTime: 1675263600, - genesisValidatorsRoot: "0x53a92d8f2bb1d85f62d16a156e6ebcd1bcaba652d0900b2c2f387826f3481f6f", - }, }; From 0c11b03effbf106dbaaecf3a610b566912a15719 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Sat, 1 Jul 2023 17:47:47 +0200 Subject: [PATCH 43/96] deps: update fastify to 4.19.0 (#5726) --- packages/api/package.json | 2 +- packages/api/src/utils/server/types.ts | 4 +- packages/beacon-node/package.json | 2 +- packages/beacon-node/src/api/impl/errors.ts | 6 +- yarn.lock | 80 +++++++++++---------- 5 files changed, 52 insertions(+), 42 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 131283eb77e7..319cad174859 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -83,7 +83,7 @@ "@types/eventsource": "^1.1.11", "@types/qs": "^6.9.7", "ajv": "^8.12.0", - "fastify": "^4.15.0" + "fastify": "^4.19.0" }, "keywords": [ "ethereum", diff --git a/packages/api/src/utils/server/types.ts b/packages/api/src/utils/server/types.ts index c4e0de749a80..0e2d5201f51c 100644 --- a/packages/api/src/utils/server/types.ts +++ b/packages/api/src/utils/server/types.ts @@ -1,12 +1,12 @@ // eslint-disable-next-line import/no-extraneous-dependencies -import type {FastifyInstance} from "fastify"; +import type {FastifyInstance, FastifyContextConfig} from "fastify"; // eslint-disable-next-line import/no-extraneous-dependencies import type * as fastify from "fastify"; import {ReqGeneric} from "../types.js"; export type ServerInstance = FastifyInstance; -export type RouteConfig = { +export type RouteConfig = FastifyContextConfig & { operationId: ServerRoute["id"]; }; diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 94182c1a71d0..a5b315d00ff8 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -139,7 +139,7 @@ "datastore-core": "^8.0.1", "datastore-level": "^9.0.1", "deepmerge": "^4.3.1", - "fastify": "^4.15.0", + "fastify": "^4.19.0", "gc-stats": "^1.4.0", "interface-datastore": "^7.0.0", "it-all": "^3.0.1", diff --git a/packages/beacon-node/src/api/impl/errors.ts b/packages/beacon-node/src/api/impl/errors.ts index 2830a0438f3b..7169b7138295 100644 --- a/packages/beacon-node/src/api/impl/errors.ts +++ b/packages/beacon-node/src/api/impl/errors.ts @@ -1,6 +1,8 @@ +import {HttpErrorCodes} from "@lodestar/api"; + export class ApiError extends Error { - statusCode: number; - constructor(statusCode: number, message?: string) { + statusCode: HttpErrorCodes; + constructor(statusCode: HttpErrorCodes, message?: string) { super(message); this.statusCode = statusCode; } diff --git a/yarn.lock b/yarn.lock index dba94ac93595..abf9763d38da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1608,17 +1608,17 @@ resolved "https://registry.yarnpkg.com/@fastify/deepmerge/-/deepmerge-1.3.0.tgz#8116858108f0c7d9fd460d05a7d637a13fe3239a" integrity sha512-J8TOSBq3SoZbDhM9+R/u77hP93gz/rajSA+K2kGyijPpORPWUXHUpTaleoj+92As0S9uPRP7Oi8IqMf0u+ro6A== -"@fastify/error@^3.0.0": - version "3.2.0" - resolved "https://registry.yarnpkg.com/@fastify/error/-/error-3.2.0.tgz#9010e0acfe07965f5fc7d2b367f58f042d0f4106" - integrity sha512-KAfcLa+CnknwVi5fWogrLXgidLic+GXnLjijXdpl8pvkvbXU5BGa37iZO9FGvsh9ZL4y+oFi5cbHBm5UOG+dmQ== +"@fastify/error@^3.2.0": + version "3.3.0" + resolved "https://registry.yarnpkg.com/@fastify/error/-/error-3.3.0.tgz#eba790082e1144bfc8def0c2c8ef350064bc537b" + integrity sha512-dj7vjIn1Ar8sVXj2yAXiMNCJDmS9MQ9XMlIecX2dIzzhjSHCyKo4DdXjXMs7wKW2kj6yvVRSpuQjOZ3YLrh56w== -"@fastify/fast-json-stringify-compiler@^4.2.0": - version "4.2.0" - resolved "https://registry.yarnpkg.com/@fastify/fast-json-stringify-compiler/-/fast-json-stringify-compiler-4.2.0.tgz#52d047fac76b0d75bd660f04a5dd606659f57c5a" - integrity sha512-ypZynRvXA3dibfPykQN3RB5wBdEUgSGgny8Qc6k163wYPLD4mEGEDkACp+00YmqkGvIm8D/xYoHajwyEdWD/eg== +"@fastify/fast-json-stringify-compiler@^4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@fastify/fast-json-stringify-compiler/-/fast-json-stringify-compiler-4.3.0.tgz#5df89fa4d1592cbb8780f78998355feb471646d5" + integrity sha512-aZAXGYo6m22Fk1zZzEUKBvut/CIIQe/BapEORnxiD5Qr0kPHqqI69NtEMCme74h+at72sPhbkb4ZrLd1W3KRLA== dependencies: - fast-json-stringify "^5.0.0" + fast-json-stringify "^5.7.0" "@gar/promisify@^1.0.1", "@gar/promisify@^1.1.3": version "1.1.3" @@ -4734,7 +4734,7 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== -avvio@^8.2.0: +avvio@^8.2.1: version "8.2.1" resolved "https://registry.yarnpkg.com/avvio/-/avvio-8.2.1.tgz#b5a482729847abb84d5aadce06511c04a0a62f82" integrity sha512-TAlMYvOuwGyLK3PfBb5WKBXZmXz2fVCgv23d6zZFdle/q3gPjmxBaeuC0pY0Dzs5PWMSgfqqEZkrye19GlDTgw== @@ -7649,7 +7649,7 @@ fast-json-stable-stringify@^2.0.0: resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== -fast-json-stringify@^5.0.0: +fast-json-stringify@^5.7.0: version "5.7.0" resolved "https://registry.yarnpkg.com/fast-json-stringify/-/fast-json-stringify-5.7.0.tgz#b0a04c848fdeb6ecd83440c71a4db35067023bed" integrity sha512-sBVPTgnAZseLu1Qgj6lUbQ0HfjFhZWXAmpZ5AaSGkyLh5gAXBga/uPJjQPHpDFjC9adWIpdOcCLSDTgrZ7snoQ== @@ -7700,26 +7700,27 @@ fastify-plugin@^4.0.0: resolved "https://registry.yarnpkg.com/fastify-plugin/-/fastify-plugin-4.5.0.tgz#8b853923a0bba6ab6921bb8f35b81224e6988d91" integrity sha512-79ak0JxddO0utAXAQ5ccKhvs6vX2MGyHHMMsmZkBANrq3hXc1CHzvNPHOcvTsVMEPl5I+NT+RO4YKMGehOfSIg== -fastify@^4.15.0: - version "4.15.0" - resolved "https://registry.yarnpkg.com/fastify/-/fastify-4.15.0.tgz#4ebadaea706217467a332341f9cfa632072d51f2" - integrity sha512-m/CaRN8nf5uyYdrDe2qqq+0z3oGyE+A++qlKQoLJTI4WI0nWK9D6R3FxXQ3MVwt/md977GMR4F43pE9oqrS2zw== +fastify@^4.19.0: + version "4.19.0" + resolved "https://registry.yarnpkg.com/fastify/-/fastify-4.19.0.tgz#2475b4cf0075aa7bd3b9651ee9fc57a1bf67cee0" + integrity sha512-Fb7w46k3kum/V+OMiw2/LN1lo2xqhJqwTsv+LGie5n7YNdA9bmDUntXfcec9rG6VRDXIBfPf5xmFv2wooBdxzg== dependencies: "@fastify/ajv-compiler" "^3.5.0" - "@fastify/error" "^3.0.0" - "@fastify/fast-json-stringify-compiler" "^4.2.0" + "@fastify/error" "^3.2.0" + "@fastify/fast-json-stringify-compiler" "^4.3.0" abstract-logging "^2.0.1" - avvio "^8.2.0" + avvio "^8.2.1" fast-content-type-parse "^1.0.0" + fast-json-stringify "^5.7.0" find-my-way "^7.6.0" - light-my-request "^5.6.1" - pino "^8.5.0" - process-warning "^2.0.0" + light-my-request "^5.9.1" + pino "^8.12.0" + process-warning "^2.2.0" proxy-addr "^2.0.7" rfdc "^1.3.0" secure-json-parse "^2.5.0" - semver "^7.3.7" - tiny-lru "^10.0.0" + semver "^7.5.0" + tiny-lru "^11.0.1" fastq@^1.6.0: version "1.13.0" @@ -10309,10 +10310,10 @@ libp2p@0.42.2: wherearewe "^2.0.0" xsalsa20 "^1.1.0" -light-my-request@^5.6.1: - version "5.9.1" - resolved "https://registry.yarnpkg.com/light-my-request/-/light-my-request-5.9.1.tgz#076f8d4cc4639408cc48381d4f2860212d469d4b" - integrity sha512-UT7pUk8jNCR1wR7w3iWfIjx32DiB2f3hFdQSOwy3/EPQ3n3VocyipUxcyRZR0ahoev+fky69uA+GejPa9KuHKg== +light-my-request@^5.9.1: + version "5.10.0" + resolved "https://registry.yarnpkg.com/light-my-request/-/light-my-request-5.10.0.tgz#0a2bbc1d1bb573ed3b78143960920ecdc05bf157" + integrity sha512-ZU2D9GmAcOUculTTdH9/zryej6n8TzT+fNGdNtm6SDp5MMMpHrJJkvAdE3c6d8d2chE9i+a//dS9CWZtisknqA== dependencies: cookie "^0.5.0" process-warning "^2.0.0" @@ -12597,10 +12598,10 @@ pino-std-serializers@^6.0.0: resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.2.0.tgz#169048c0df3f61352fce56aeb7fb962f1b66ab43" integrity sha512-IWgSzUL8X1w4BIWTwErRgtV8PyOGOOi60uqv0oKuS/fOA8Nco/OeI6lBuc4dyP8MMfdFwyHqTMcBIA7nDiqEqA== -pino@^8.5.0: - version "8.11.0" - resolved "https://registry.yarnpkg.com/pino/-/pino-8.11.0.tgz#2a91f454106b13e708a66c74ebc1c2ab7ab38498" - integrity sha512-Z2eKSvlrl2rH8p5eveNUnTdd4AjJk8tAsLkHYZQKGHP4WTh2Gi1cOSOs3eWPqaj+niS3gj4UkoreoaWgF3ZWYg== +pino@^8.12.0: + version "8.14.1" + resolved "https://registry.yarnpkg.com/pino/-/pino-8.14.1.tgz#bb38dcda8b500dd90c1193b6c9171eb777a47ac8" + integrity sha512-8LYNv7BKWXSfS+k6oEc6occy5La+q2sPwU3q2ljTX5AZk7v+5kND2o5W794FyRaqha6DJajmkNRsWtPpFyMUdw== dependencies: atomic-sleep "^1.0.0" fast-redact "^3.1.1" @@ -12707,7 +12708,7 @@ process-on-spawn@^1.0.0: dependencies: fromentries "^1.2.0" -process-warning@^2.0.0: +process-warning@^2.0.0, process-warning@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-2.2.0.tgz#008ec76b579820a8e5c35d81960525ca64feb626" integrity sha512-/1WZ8+VQjR6avWOgHeEPd7SDQmFQ1B5mC1eRXsCm5TarlNmx/wCsa5GEaxGm05BORRtyG/Ex/3xq3TuRvq57qg== @@ -13635,6 +13636,13 @@ semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semve dependencies: lru-cache "^6.0.0" +semver@^7.5.0: + version "7.5.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" + integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== + dependencies: + lru-cache "^6.0.0" + send@0.18.0: version "0.18.0" resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" @@ -14644,10 +14652,10 @@ timers-browserify@^2.0.4: dependencies: setimmediate "^1.0.4" -tiny-lru@^10.0.0: - version "10.4.1" - resolved "https://registry.yarnpkg.com/tiny-lru/-/tiny-lru-10.4.1.tgz#dec67a62115a4cb31d2065b8116d010daac362fe" - integrity sha512-buLIzw7ppqymuO3pt10jHk/6QMeZLbidihMQU+N6sogF6EnBzG0qtDWIHuhw1x3dyNgVL/KTGIZsTK81+yCzLg== +tiny-lru@^11.0.1: + version "11.0.1" + resolved "https://registry.yarnpkg.com/tiny-lru/-/tiny-lru-11.0.1.tgz#629d6ddd88bd03c0929722680167f1feadf576f2" + integrity sha512-iNgFugVuQgBKrqeO/mpiTTgmBsTP0WL6yeuLfLs/Ctf0pI/ixGqIRm8sDCwMcXGe9WWvt2sGXI5mNqZbValmJg== "tiny-worker@>= 2": version "2.3.0" From 0f0f533e1ac6163082eb0b7baa7499c4e02ae568 Mon Sep 17 00:00:00 2001 From: g11tech Date: Sat, 1 Jul 2023 21:32:03 +0530 Subject: [PATCH 44/96] feat: deprecate engine_exchangeTransitionConfigurationV1 (#5724) --- dashboards/lodestar_execution_engine.json | 118 ------------------ packages/beacon-node/src/chain/chain.ts | 65 +--------- .../src/execution/engine/disabled.ts | 4 - .../beacon-node/src/execution/engine/http.ts | 21 ---- .../src/execution/engine/interface.ts | 4 - .../beacon-node/src/execution/engine/mock.ts | 8 -- .../beacon-node/src/execution/engine/types.ts | 16 +-- 7 files changed, 2 insertions(+), 234 deletions(-) diff --git a/dashboards/lodestar_execution_engine.json b/dashboards/lodestar_execution_engine.json index 58a89801b91a..698480f19355 100644 --- a/dashboards/lodestar_execution_engine.json +++ b/dashboards/lodestar_execution_engine.json @@ -1792,124 +1792,6 @@ "title": "forkchoiceUpdatedV2", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "µs" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "0.9999" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 59 - }, - "id": 487, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rpc_duration_engine_exchangeTransitionConfigurationV1_success", - "legendFormat": "{{quantile}}", - "range": true, - "refId": "A" - } - ], - "title": "exchangeTransitionConfigurationV1", - "type": "timeseries" - }, { "collapsed": false, "datasource": { diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 349b7a95d4f7..ccb2c023c5fb 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -34,10 +34,8 @@ import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js"; import {IBeaconDb} from "../db/index.js"; import {Metrics} from "../metrics/index.js"; -import {bytesToData, numToQuantity} from "../eth1/provider/utils.js"; -import {wrapError} from "../util/wrapError.js"; import {IEth1ForBlockProduction} from "../eth1/index.js"; -import {IExecutionEngine, IExecutionBuilder, TransitionConfigurationV1} from "../execution/index.js"; +import {IExecutionEngine, IExecutionBuilder} from "../execution/index.js"; import {Clock, ClockEvent, IClock} from "../util/clock.js"; import {ensureDir, writeIfNotExist} from "../util/file.js"; import {isOptimisticBlock} from "../util/forkChoice.js"; @@ -143,9 +141,6 @@ export class BeaconChain implements IBeaconChain { protected readonly db: IBeaconDb; private readonly archiver: Archiver; private abortController = new AbortController(); - private successfulExchangeTransition = false; - private readonly exchangeTransitionConfigurationEverySlots: number; - private processShutdownCallback: ProcessShutdownCallback; constructor( @@ -187,11 +182,6 @@ export class BeaconChain implements IBeaconChain { this.eth1 = eth1; this.executionEngine = executionEngine; this.executionBuilder = executionBuilder; - // From https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#specification-3 - // > Consensus Layer client software SHOULD poll this endpoint every 60 seconds. - // Align to a multiple of SECONDS_PER_SLOT for nicer logs - this.exchangeTransitionConfigurationEverySlots = Math.floor(60 / this.config.SECONDS_PER_SLOT); - const signal = this.abortController.signal; const emitter = new ChainEventEmitter(); // by default, verify signatures on both main threads and worker threads @@ -778,13 +768,6 @@ export class BeaconChain implements IBeaconChain { this.seenAttestationDatas.onSlot(slot); this.reprocessController.onSlot(slot); - if (isFinite(this.config.BELLATRIX_FORK_EPOCH) && slot % this.exchangeTransitionConfigurationEverySlots === 0) { - this.exchangeTransitionConfiguration().catch((e) => { - // Should never throw - this.logger.error("Error on exchangeTransitionConfiguration", {}, e as Error); - }); - } - // Prune old blobSidecars for block production, those are only useful on their slot if (this.config.getForkSeq(slot) >= ForkSeq.deneb) { if (this.producedBlobSidecarsCache.size > 0) { @@ -857,52 +840,6 @@ export class BeaconChain implements IBeaconChain { } } - /** - * perform heart beat for EL lest it logs warning that CL is not connected - */ - private async exchangeTransitionConfiguration(): Promise { - const clConfig: TransitionConfigurationV1 = { - terminalTotalDifficulty: numToQuantity(this.config.TERMINAL_TOTAL_DIFFICULTY), - terminalBlockHash: bytesToData(this.config.TERMINAL_BLOCK_HASH), - /** terminalBlockNumber has to be set to zero for now as per specs */ - terminalBlockNumber: numToQuantity(0), - }; - - const elConfigRes = await wrapError(this.executionEngine.exchangeTransitionConfigurationV1(clConfig)); - - if (elConfigRes.err) { - // Note: Will throw an error if: - // - EL endpoint is offline, unreachable, port not exposed, etc - // - JWT secret is not properly configured - // - If there is a missmatch in configuration with Geth, see https://github.com/ethereum/go-ethereum/blob/0016eb7eeeb42568c8c20d0cb560ddfc9a938fad/eth/catalyst/api.go#L301 - this.successfulExchangeTransition = false; - - this.logger.warn("Could not validate transition configuration with execution client", {}, elConfigRes.err); - } else { - // Note: This code is useless when connected to Geth. If there's a configuration mismatch Geth returns an - // error instead of its own transition configuration, so we can't do this comparision. - const elConfig = elConfigRes.result; - const keysToCheck: (keyof TransitionConfigurationV1)[] = ["terminalTotalDifficulty", "terminalBlockHash"]; - const errors: string[] = []; - - for (const key of keysToCheck) { - if (elConfig[key] !== clConfig[key]) { - errors.push(`different ${key} (cl ${clConfig[key]} el ${elConfig[key]})`); - } - } - - if (errors.length > 0) { - this.logger.warn(`Transition configuration mismatch: ${errors.join(", ")}`); - } else { - // Only log once per successful call - if (!this.successfulExchangeTransition) { - this.logger.info("Validated transition configuration with execution client", clConfig); - this.successfulExchangeTransition = true; - } - } - } - } - async updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise { proposers.forEach((proposer) => { this.beaconProposerCache.add(epoch, proposer); diff --git a/packages/beacon-node/src/execution/engine/disabled.ts b/packages/beacon-node/src/execution/engine/disabled.ts index 45c4a135c2a8..6c6804e970c3 100644 --- a/packages/beacon-node/src/execution/engine/disabled.ts +++ b/packages/beacon-node/src/execution/engine/disabled.ts @@ -19,10 +19,6 @@ export class ExecutionEngineDisabled implements IExecutionEngine { throw Error("Execution engine disabled"); } - async exchangeTransitionConfigurationV1(): Promise { - throw Error("Execution engine disabled"); - } - getPayloadBodiesByHash(): Promise { throw Error("Execution engine disabled"); } diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index 35c54a96418e..3cf9f46297fe 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -13,7 +13,6 @@ import { IExecutionEngine, PayloadId, PayloadAttributes, - TransitionConfigurationV1, BlobsBundle, VersionedHashes, } from "./interface.js"; @@ -72,7 +71,6 @@ const QUEUE_MAX_LENGTH = EPOCHS_PER_BATCH * SLOTS_PER_EPOCH * 2; const notifyNewPayloadOpts: ReqOpts = {routeId: "notifyNewPayload"}; const forkchoiceUpdatedV1Opts: ReqOpts = {routeId: "forkchoiceUpdated"}; const getPayloadOpts: ReqOpts = {routeId: "getPayload"}; -const exchageTransitionConfigOpts: ReqOpts = {routeId: "exchangeTransitionConfiguration"}; /** * based on Ethereum JSON-RPC API and inherits the following properties of this standard: @@ -345,25 +343,6 @@ export class ExecutionEngineHttp implements IExecutionEngine { return parseExecutionPayload(fork, payloadResponse); } - /** - * `engine_exchangeTransitionConfigurationV1` - * - * An api method for EL<>CL transition config matching and heartbeat - */ - - async exchangeTransitionConfigurationV1( - transitionConfiguration: TransitionConfigurationV1 - ): Promise { - const method = "engine_exchangeTransitionConfigurationV1"; - return this.rpc.fetchWithRetries( - { - method, - params: [transitionConfiguration], - }, - exchageTransitionConfigOpts - ); - } - async prunePayloadIdCache(): Promise { this.payloadIdCache.prune(); } diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index c77c80a22e54..0b2d96229ced 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -129,10 +129,6 @@ export interface IExecutionEngine { payloadId: PayloadId ): Promise<{executionPayload: allForks.ExecutionPayload; blockValue: Wei; blobsBundle?: BlobsBundle}>; - exchangeTransitionConfigurationV1( - transitionConfiguration: TransitionConfigurationV1 - ): Promise; - getPayloadBodiesByHash(blockHash: DATA[]): Promise<(ExecutionPayloadBody | null)[]>; getPayloadBodiesByRange(start: number, count: number): Promise<(ExecutionPayloadBody | null)[]>; diff --git a/packages/beacon-node/src/execution/engine/mock.ts b/packages/beacon-node/src/execution/engine/mock.ts index 040cfa80798d..3e3d3d65f901 100644 --- a/packages/beacon-node/src/execution/engine/mock.ts +++ b/packages/beacon-node/src/execution/engine/mock.ts @@ -93,7 +93,6 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { engine_getPayloadV1: this.getPayload.bind(this), engine_getPayloadV2: this.getPayload.bind(this), engine_getPayloadV3: this.getPayload.bind(this), - engine_exchangeTransitionConfigurationV1: this.exchangeTransitionConfigurationV1.bind(this), engine_getPayloadBodiesByHashV1: this.getPayloadBodiesByHash.bind(this), engine_getPayloadBodiesByRangeV1: this.getPayloadBodiesByRange.bind(this), }; @@ -388,13 +387,6 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { return payload.executionPayload; } - private exchangeTransitionConfigurationV1( - transitionConfiguration: EngineApiRpcParamTypes["engine_exchangeTransitionConfigurationV1"][0] - ): EngineApiRpcReturnTypes["engine_exchangeTransitionConfigurationV1"] { - // echo same configuration from consensus, which will be considered valid - return transitionConfiguration; - } - private timestampToFork(timestamp: number): ForkExecution { if (timestamp > (this.opts.denebForkTimestamp ?? Infinity)) return ForkName.deneb; if (timestamp > (this.opts.capellaForkTimestamp ?? Infinity)) return ForkName.capella; diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 55f9e07b737a..7295e8c37575 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -16,13 +16,7 @@ import { QUANTITY, quantityToBigint, } from "../../eth1/provider/utils.js"; -import { - ExecutePayloadStatus, - TransitionConfigurationV1, - BlobsBundle, - PayloadAttributes, - VersionedHashes, -} from "./interface.js"; +import {ExecutePayloadStatus, BlobsBundle, PayloadAttributes, VersionedHashes} from "./interface.js"; import {WithdrawalV1} from "./payloadIdCache.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -53,10 +47,6 @@ export type EngineApiRpcParamTypes = { engine_getPayloadV1: [QUANTITY]; engine_getPayloadV2: [QUANTITY]; engine_getPayloadV3: [QUANTITY]; - /** - * 1. Object - Instance of TransitionConfigurationV1 - */ - engine_exchangeTransitionConfigurationV1: [TransitionConfigurationV1]; /** * 1. Array of DATA - Array of block_hash field values of the ExecutionPayload structure @@ -98,10 +88,6 @@ export type EngineApiRpcReturnTypes = { engine_getPayloadV1: ExecutionPayloadRpc; engine_getPayloadV2: ExecutionPayloadResponse; engine_getPayloadV3: ExecutionPayloadResponse; - /** - * Object - Instance of TransitionConfigurationV1 - */ - engine_exchangeTransitionConfigurationV1: TransitionConfigurationV1; engine_getPayloadBodiesByHashV1: (ExecutionPayloadBodyRpc | null)[]; From 9335cc552bda504ffe67752b4076784df2d11cc3 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 4 Jul 2023 15:52:47 +0200 Subject: [PATCH 45/96] chore: disable rate limiting on local testnet node (#5728) --- scripts/dev/node1.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/dev/node1.sh b/scripts/dev/node1.sh index 4260458b4d97..90153ecc7d1b 100755 --- a/scripts/dev/node1.sh +++ b/scripts/dev/node1.sh @@ -13,4 +13,5 @@ GENESIS_TIME=$(date +%s) --rest.namespace '*' \ --metrics \ --logLevel debug \ - --eth1 false + --eth1 false \ + --network.rateLimitMultiplier 0 From 7280234bea66b49da3900b916a1b54c4666e4173 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 4 Jul 2023 21:37:44 -0400 Subject: [PATCH 46/96] deps: update libp2p to 0.45.x (#5506) * deps: update libp2p * Fix libp2p components * Fix e2e tests * fix merge error * chore: bump libp2p to 0.45.6 * chore: fix yarn lock * Update to gossipsub 9.0.0 * Fix linter error * Rely entirely on lodestar's peer manager to prune connections * Update libp2p and gossipsub * Remove unused Libp2pEvent members --- package.json | 2 +- packages/beacon-node/package.json | 43 +- packages/beacon-node/src/constants/network.ts | 6 +- .../src/network/core/networkCore.ts | 4 +- .../src/network/gossip/gossipsub.ts | 6 +- packages/beacon-node/src/network/interface.ts | 22 +- .../beacon-node/src/network/nodejs/bundle.ts | 69 +- .../beacon-node/src/network/nodejs/noise.ts | 4 +- .../src/network/peers/datastore.ts | 5 +- .../beacon-node/src/network/peers/discover.ts | 19 +- .../src/network/peers/peerManager.ts | 29 +- .../peers/utils/getConnectedPeerIds.ts | 4 +- packages/beacon-node/src/network/util.ts | 22 +- .../test/e2e/network/gossipsub.test.ts | 2 +- .../e2e/network/peers/peerManager.test.ts | 11 +- packages/cli/package.json | 4 +- packages/db/package.json | 2 +- packages/reqresp/package.json | 10 +- .../reqresp/src/encoders/requestDecode.ts | 4 +- .../reqresp/test/unit/response/index.test.ts | 2 +- packages/reqresp/test/utils/index.ts | 7 +- types/it-pipe/index.d.ts | 8 +- yarn.lock | 1486 +++++++---------- 23 files changed, 744 insertions(+), 1027 deletions(-) diff --git a/package.json b/package.json index 2fe8168b3e1d..10a8c70bb391 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "karma-spec-reporter": "^0.0.36", "karma-webpack": "^5.0.0", "lerna": "^6.6.1", - "libp2p": "0.42.2", + "libp2p": "0.45.9", "mocha": "^10.2.0", "node-gyp": "^9.3.1", "npm-run-all": "^4.1.5", diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 55f63b0c3361..fbdb5aa9609b 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -98,9 +98,9 @@ "@chainsafe/as-sha256": "^0.3.1", "@chainsafe/bls": "7.1.1", "@chainsafe/blst": "^0.2.9", - "@chainsafe/discv5": "^3.0.0", - "@chainsafe/libp2p-gossipsub": "^6.2.0", - "@chainsafe/libp2p-noise": "^11.0.4", + "@chainsafe/discv5": "^4.0.0", + "@chainsafe/libp2p-gossipsub": "^9.1.0", + "@chainsafe/libp2p-noise": "^12.0.1", "@chainsafe/persistent-merkle-tree": "^0.5.0", "@chainsafe/prometheus-gc-stats": "^1.0.0", "@chainsafe/ssz": "^0.10.2", @@ -108,17 +108,17 @@ "@ethersproject/abi": "^5.7.0", "@fastify/bearer-auth": "^9.0.0", "@fastify/cors": "^8.2.1", - "@libp2p/bootstrap": "^6.0.3", - "@libp2p/interface-connection": "^3.0.2", - "@libp2p/interface-connection-manager": "^1.3.0", - "@libp2p/interface-peer-id": "^2.0.1", - "@libp2p/interface-pubsub": "^3.0.0", - "@libp2p/interface-registrar": "^2.0.8", - "@libp2p/mdns": "^6.0.0", - "@libp2p/mplex": "^7.1.3", - "@libp2p/peer-id-factory": "^2.0.1", - "@libp2p/prometheus-metrics": "^1.1.3", - "@libp2p/tcp": "6.1.0", + "@libp2p/bootstrap": "^8.0.0", + "@libp2p/interface-connection": "^5.1.0", + "@libp2p/interface-connection-manager": "^3.0.1", + "@libp2p/interface-peer-id": "^2.0.2", + "@libp2p/interface-pubsub": "^4.0.1", + "@libp2p/interface-registrar": "^2.0.12", + "@libp2p/mdns": "^8.0.0", + "@libp2p/mplex": "^8.0.3", + "@libp2p/peer-id-factory": "^2.0.3", + "@libp2p/prometheus-metrics": "^1.1.4", + "@libp2p/tcp": "7.0.1", "@lodestar/api": "^1.9.1", "@lodestar/config": "^1.9.1", "@lodestar/db": "^1.9.1", @@ -131,21 +131,20 @@ "@lodestar/types": "^1.9.1", "@lodestar/utils": "^1.9.1", "@lodestar/validator": "^1.9.1", - "@multiformats/multiaddr": "^11.0.0", + "@multiformats/multiaddr": "^12.1.3", "@types/datastore-level": "^3.0.0", "buffer-xor": "^2.0.2", "c-kzg": "^2.1.0", "cross-fetch": "^3.1.4", - "datastore-core": "^8.0.1", - "datastore-level": "^9.0.1", + "datastore-core": "^9.1.1", + "datastore-level": "^10.1.1", "deepmerge": "^4.3.1", "fastify": "^4.19.0", - "gc-stats": "^1.4.0", - "interface-datastore": "^7.0.0", - "it-all": "^3.0.1", - "it-pipe": "^2.0.5", + "interface-datastore": "^8.2.0", + "it-all": "^3.0.2", + "it-pipe": "^3.0.1", "jwt-simple": "0.5.6", - "libp2p": "0.42.2", + "libp2p": "0.45.9", "prom-client": "^14.2.0", "qs": "^6.11.1", "snappyjs": "^0.7.0", diff --git a/packages/beacon-node/src/constants/network.ts b/packages/beacon-node/src/constants/network.ts index 9c59426e8cee..f0282893e8f4 100644 --- a/packages/beacon-node/src/constants/network.ts +++ b/packages/beacon-node/src/constants/network.ts @@ -43,8 +43,8 @@ export const GOODBYE_KNOWN_CODES: Record = { 251: "Peer banned this node", }; -/** Until js-libp2p types its events */ +/** Until js-libp2p exports an enum for its events */ export enum Libp2pEvent { - peerConnect = "peer:connect", - peerDisconnect = "peer:disconnect", + connectionOpen = "connection:open", + connectionClose = "connection:close", } diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts index 4eb87d438c04..ee29d37e8efe 100644 --- a/packages/beacon-node/src/network/core/networkCore.ts +++ b/packages/beacon-node/src/network/core/networkCore.ts @@ -353,7 +353,7 @@ export class NetworkCore implements INetworkCore { } getConnectionsByPeer(): Map { - return getConnectionsMap(this.libp2p.connectionManager); + return getConnectionsMap(this.libp2p); } async getConnectedPeers(): Promise { @@ -368,7 +368,7 @@ export class NetworkCore implements INetworkCore { async connectToPeer(peerIdStr: PeerIdStr, multiaddrStrArr: MultiaddrStr[]): Promise { const peer = peerIdFromString(peerIdStr); - await this.libp2p.peerStore.addressBook.add(peer, multiaddrStrArr.map(multiaddr)); + await this.libp2p.peerStore.merge(peer, {multiaddrs: multiaddrStrArr.map(multiaddr)}); await this.libp2p.dial(peer); } diff --git a/packages/beacon-node/src/network/gossip/gossipsub.ts b/packages/beacon-node/src/network/gossip/gossipsub.ts index ef159e019077..649d8a0f9361 100644 --- a/packages/beacon-node/src/network/gossip/gossipsub.ts +++ b/packages/beacon-node/src/network/gossip/gossipsub.ts @@ -7,7 +7,6 @@ import {ATTESTATION_SUBNET_COUNT, ForkName, SYNC_COMMITTEE_SUBNET_COUNT} from "@ import {Logger, Map2d, Map2dArr} from "@lodestar/utils"; import {RegistryMetricCreator} from "../../metrics/index.js"; -import {peerIdFromString} from "../../util/peerId.js"; import {PeersData} from "../peers/peersData.js"; import {ClientKind} from "../peers/client.js"; import {GOSSIP_MAX_SIZE, GOSSIP_MAX_SIZE_BELLATRIX} from "../../constants/network.js"; @@ -89,7 +88,7 @@ export class Eth2Gossipsub extends GossipSub { // Gossipsub parameters defined here: // https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub - super(modules.libp2p, { + super(modules.libp2p.services.components, { globalSignaturePolicy: SignaturePolicy.StrictNoSign, allowPublishToZeroPeers: allowPublishToZeroPeers, D: gossipsubD ?? GOSSIP_D, @@ -300,8 +299,7 @@ export class Eth2Gossipsub extends GossipSub { // Without this we'll have huge event loop lag // See https://github.com/ChainSafe/lodestar/issues/5604 setTimeout(() => { - // TODO: reportMessageValidationResult should take PeerIdStr since it only uses string version - this.reportMessageValidationResult(data.msgId, peerIdFromString(data.propagationSource), data.acceptance); + this.reportMessageValidationResult(data.msgId, data.propagationSource, data.acceptance); }, 0); } } diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index 5f7ec83c76d6..9ef305c02d8f 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -1,7 +1,6 @@ import {Libp2p as ILibp2p} from "libp2p"; import {Connection} from "@libp2p/interface-connection"; -import {Registrar} from "@libp2p/interface-registrar"; -import {ConnectionManager} from "@libp2p/interface-connection-manager"; +import {Components} from "libp2p/components"; import {Slot, SlotRootHex, allForks, altair, capella, deneb, phase0} from "@lodestar/types"; import {PeerIdStr} from "../util/peerId.js"; import {INetworkEventBus} from "./events.js"; @@ -66,4 +65,21 @@ export interface INetwork extends INetworkCorePublic { export type PeerDirection = Connection["stat"]["direction"]; export type PeerStatus = Connection["stat"]["status"]; -export type Libp2p = ILibp2p & {connectionManager: ConnectionManager; registrar: Registrar}; +export type LodestarComponents = Pick< + Components, + | "peerId" + | "events" + | "addressManager" + | "peerStore" + | "upgrader" + | "registrar" + | "connectionManager" + | "transportManager" + | "connectionGater" + | "contentRouting" + | "peerRouting" + | "datastore" + | "connectionProtector" + | "metrics" +>; +export type Libp2p = ILibp2p<{components: LodestarComponents}>; diff --git a/packages/beacon-node/src/network/nodejs/bundle.ts b/packages/beacon-node/src/network/nodejs/bundle.ts index 01805af9ef74..6243ffd0364b 100644 --- a/packages/beacon-node/src/network/nodejs/bundle.ts +++ b/packages/beacon-node/src/network/nodejs/bundle.ts @@ -1,4 +1,5 @@ import {createLibp2p} from "libp2p"; +import {identifyService} from "libp2p/identify"; import {tcp} from "@libp2p/tcp"; import {mplex} from "@libp2p/mplex"; import {bootstrap} from "@libp2p/bootstrap"; @@ -42,7 +43,7 @@ export async function createNodejsLibp2p(options: Libp2pOptions): Promise maxConnections calls _maybeDisconnectOne() which will sort peers disconnect - // the one with the least `_peerValues`. That's a custom peer generalized score that's not used, so it always - // has the same value in current Lodestar usage. - maxConnections: options.maxConnections, - // DOCS: the minimum number of connections below which libp2p not activate preemptive disconnections. - // If ConnectionManager.size < minConnections, it won't prune peers in _maybeDisconnectOne(). If autoDial is - // off it doesn't have any effect in behaviour. - minConnections: options.minConnections, + // Rely entirely on lodestar's peer manager to prune connections + //maxConnections: options.maxConnections, + // DOCS: There is no way to turn off autodial other than setting minConnections to 0 + minConnections: 0, }, datastore: options.datastore, - nat: { - // libp2p usage of nat-api is broken as shown in this issue. https://github.com/ChainSafe/lodestar/issues/2996 - // Also, unnsolicited usage of UPnP is not great, and should be customizable with flags - enabled: false, - }, - relay: { - enabled: false, - hop: { - enabled: false, - active: false, - }, - advertise: { - enabled: false, - ttl: 0, - bootDelay: 0, - }, - autoRelay: { - enabled: false, - maxListeners: 0, - }, - }, - - identify: { - host: { + services: { + identify: identifyService({ agentVersion: options.hideAgentVersion ? "" : options.lodestarVersion ? `lodestar/${options.lodestarVersion}` : "lodestar", - }, + }), + // individual components are specified because the components object is a Proxy + // and passing it here directly causes problems downstream, not to mention is slowwww + components: (components: Components) => ({ + peerId: components.peerId, + events: components.events, + addressManager: components.addressManager, + peerStore: components.peerStore, + upgrader: components.upgrader, + registrar: components.registrar, + connectionManager: components.connectionManager, + transportManager: components.transportManager, + connectionGater: components.connectionGater, + contentRouting: components.contentRouting, + peerRouting: components.peerRouting, + datastore: components.datastore, + connectionProtector: components.connectionProtector, + metrics: components.metrics, + }), }, - })) as Libp2p; + }); } diff --git a/packages/beacon-node/src/network/nodejs/noise.ts b/packages/beacon-node/src/network/nodejs/noise.ts index 24574767d809..14e91c86613b 100644 --- a/packages/beacon-node/src/network/nodejs/noise.ts +++ b/packages/beacon-node/src/network/nodejs/noise.ts @@ -1,6 +1,6 @@ import type {ConnectionEncrypter} from "@libp2p/interface-connection-encrypter"; import {newInstance, ChaCha20Poly1305} from "@chainsafe/as-chacha20poly1305"; -import {ICryptoInterface, noise, stablelib} from "@chainsafe/libp2p-noise"; +import {ICryptoInterface, noise, pureJsCrypto} from "@chainsafe/libp2p-noise"; import {digest} from "@chainsafe/as-sha256"; type Bytes = Uint8Array; @@ -11,7 +11,7 @@ const asImpl = new ChaCha20Poly1305(ctx); // same to stablelib but we use as-chacha20poly1305 and as-sha256 const lodestarCrypto: ICryptoInterface = { - ...stablelib, + ...pureJsCrypto, hashSHA256(data: Uint8Array): Uint8Array { return digest(data); }, diff --git a/packages/beacon-node/src/network/peers/datastore.ts b/packages/beacon-node/src/network/peers/datastore.ts index 272bafbfb68a..762e99dc45b4 100644 --- a/packages/beacon-node/src/network/peers/datastore.ts +++ b/packages/beacon-node/src/network/peers/datastore.ts @@ -57,7 +57,7 @@ export class Eth2PeerDataStore extends BaseDatastore { return this._dbDatastore.close(); } - async put(key: Key, val: Uint8Array): Promise { + async put(key: Key, val: Uint8Array): Promise { return this._put(key, val, false); } @@ -65,7 +65,7 @@ export class Eth2PeerDataStore extends BaseDatastore { * Same interface to put with "fromDb" option, if this item is updated back from db * Move oldest items from memory data store to db if it's over this._maxMemoryItems */ - async _put(key: Key, val: Uint8Array, fromDb = false): Promise { + async _put(key: Key, val: Uint8Array, fromDb = false): Promise { while (this._memoryDatastore.size >= this._maxMemoryItems) { // it's likely this is called only 1 time await this.pruneMemoryDatastore(); @@ -83,6 +83,7 @@ export class Eth2PeerDataStore extends BaseDatastore { } if (!fromDb) await this._addDirtyItem(keyStr); + return key; } /** diff --git a/packages/beacon-node/src/network/peers/discover.ts b/packages/beacon-node/src/network/peers/discover.ts index 07738ccb11f1..8a5505a6580e 100644 --- a/packages/beacon-node/src/network/peers/discover.ts +++ b/packages/beacon-node/src/network/peers/discover.ts @@ -9,7 +9,7 @@ import {LoggerNode} from "@lodestar/logger/node"; import {NetworkCoreMetrics} from "../core/metrics.js"; import {Libp2p} from "../interface.js"; import {ENRKey, SubnetType} from "../metadata.js"; -import {getConnectionsMap, getDefaultDialer, prettyPrintPeerId} from "../util.js"; +import {getConnectionsMap, prettyPrintPeerId} from "../util.js"; import {Discv5Worker} from "../discv5/index.js"; import {LodestarDiscv5Opts} from "../discv5/types.js"; import {deserializeEnrSubnets, zeroAttnets, zeroSyncnets} from "./utils/enrSubnetsDeserialize.js"; @@ -163,12 +163,17 @@ export class PeerDiscovery { const cachedENRsToDial = new Map(); // Iterate in reverse to consider first the most recent ENRs const cachedENRsReverse: CachedENR[] = []; + const pendingDials = new Set( + this.libp2p.services.components.connectionManager + .getDialQueue() + .map((pendingDial) => pendingDial.peerId?.toString()) + ); for (const [id, cachedENR] of this.cachedENRs.entries()) { if ( // time expired or Date.now() - cachedENR.addedUnixMs > MAX_CACHED_ENR_AGE_MS || // already dialing - getDefaultDialer(this.libp2p).pendingDials.has(id) + pendingDials.has(id) ) { this.cachedENRs.delete(id); } else { @@ -324,7 +329,11 @@ export class PeerDiscovery { } // Ignore dialing peers - if (getDefaultDialer(this.libp2p).pendingDials.has(peerId.toString())) { + if ( + this.libp2p.services.components.connectionManager + .getDialQueue() + .find((pendingDial) => pendingDial.peerId && pendingDial.peerId.equals(peerId)) + ) { return DiscoveredPeerStatus.already_dialing; } @@ -391,7 +400,7 @@ export class PeerDiscovery { // Must add the multiaddrs array to the address book before dialing // https://github.com/libp2p/js-libp2p/blob/aec8e3d3bb1b245051b60c2a890550d262d5b062/src/index.js#L638 - await this.libp2p.peerStore.addressBook.add(peerId, [multiaddrTCP]); + await this.libp2p.peerStore.merge(peerId, {multiaddrs: [multiaddrTCP]}); // Note: PeerDiscovery adds the multiaddrTCP beforehand const peerIdShort = prettyPrintPeerId(peerId); @@ -415,7 +424,7 @@ export class PeerDiscovery { /** Check if there is 1+ open connection with this peer */ private isPeerConnected(peerIdStr: PeerIdStr): boolean { - const connections = getConnectionsMap(this.libp2p.connectionManager).get(peerIdStr); + const connections = getConnectionsMap(this.libp2p).get(peerIdStr); return Boolean(connections && connections.some((connection) => connection.stat.status === "OPEN")); } } diff --git a/packages/beacon-node/src/network/peers/peerManager.ts b/packages/beacon-node/src/network/peers/peerManager.ts index a40610584919..4b572f425c16 100644 --- a/packages/beacon-node/src/network/peers/peerManager.ts +++ b/packages/beacon-node/src/network/peers/peerManager.ts @@ -170,8 +170,8 @@ export class PeerManager { metrics.peers.addCollect(() => this.runPeerCountMetrics(metrics)); } - this.libp2p.connectionManager.addEventListener(Libp2pEvent.peerConnect, this.onLibp2pPeerConnect); - this.libp2p.connectionManager.addEventListener(Libp2pEvent.peerDisconnect, this.onLibp2pPeerDisconnect); + this.libp2p.services.components.events.addEventListener(Libp2pEvent.connectionOpen, this.onLibp2pPeerConnect); + this.libp2p.services.components.events.addEventListener(Libp2pEvent.connectionClose, this.onLibp2pPeerDisconnect); this.networkEventBus.on(NetworkEvent.reqRespRequest, this.onRequest); // On start-up will connected to existing peers in libp2p.peerStore, same as autoDial behaviour @@ -202,8 +202,11 @@ export class PeerManager { async close(): Promise { await this.discovery?.stop(); - this.libp2p.connectionManager.removeEventListener(Libp2pEvent.peerConnect, this.onLibp2pPeerConnect); - this.libp2p.connectionManager.removeEventListener(Libp2pEvent.peerDisconnect, this.onLibp2pPeerDisconnect); + this.libp2p.services.components.events.removeEventListener(Libp2pEvent.connectionOpen, this.onLibp2pPeerConnect); + this.libp2p.services.components.events.removeEventListener( + Libp2pEvent.connectionClose, + this.onLibp2pPeerDisconnect + ); this.networkEventBus.off(NetworkEvent.reqRespRequest, this.onRequest); for (const interval of this.intervals) clearInterval(interval); } @@ -320,7 +323,7 @@ export class PeerManager { this.logger.verbose("Received goodbye request", {peer: prettyPrintPeerId(peer), goodbye, reason}); this.metrics?.peerGoodbyeReceived.inc({reason}); - const conn = getConnection(this.libp2p.connectionManager, peer.toString()); + const conn = getConnection(this.libp2p, peer.toString()); if (conn && Date.now() - conn.stat.timeline.open > LONG_PEER_CONNECTION_MS) { this.metrics?.peerLongConnectionDisconnect.inc({reason}); } @@ -366,12 +369,14 @@ export class PeerManager { // libp2p.connectionManager.get() returns not null if there's +1 open connections with `peer` if (peerData && peerData.relevantStatus !== RelevantPeerStatus.relevant) { this.libp2p.peerStore - // ttl = undefined means it's never expired - .tagPeer(peer, PEER_RELEVANT_TAG, {ttl: undefined, value: PEER_RELEVANT_TAG_VALUE}) + .merge(peer, { + // ttl = undefined means it's never expired + tags: {[PEER_RELEVANT_TAG]: {ttl: undefined, value: PEER_RELEVANT_TAG_VALUE}}, + }) .catch((e) => this.logger.verbose("cannot tag peer", {peerId: peer.toString()}, e as Error)); peerData.relevantStatus = RelevantPeerStatus.relevant; } - if (getConnection(this.libp2p.connectionManager, peer.toString())) { + if (getConnection(this.libp2p, peer.toString())) { this.networkEventBus.emit(NetworkEvent.peerConnected, {peer: peer.toString(), status}); } } @@ -608,7 +613,7 @@ export class PeerManager { // since it's not possible to handle it async, we have to wait for a while to set AgentVersion // See https://github.com/libp2p/js-libp2p/pull/1168 setTimeout(async () => { - const agentVersionBytes = await this.libp2p.peerStore.metadataBook.getValue(peerData.peerId, "AgentVersion"); + const agentVersionBytes = (await this.libp2p.peerStore.get(peerData.peerId)).metadata.get("AgentVersion"); if (agentVersionBytes) { const agentVersion = new TextDecoder().decode(agentVersionBytes) || "N/A"; peerData.agentVersion = agentVersion; @@ -632,7 +637,7 @@ export class PeerManager { this.networkEventBus.emit(NetworkEvent.peerDisconnected, {peer: peer.toString()}); this.metrics?.peerDisconnectedEvent.inc({direction}); this.libp2p.peerStore - .unTagPeer(peer, PEER_RELEVANT_TAG) + .merge(peer, {tags: {[PEER_RELEVANT_TAG]: undefined}}) .catch((e) => this.logger.verbose("cannot untag peer", {peerId: peer.toString()}, e as Error)); }; @@ -649,7 +654,7 @@ export class PeerManager { const reason = GOODBYE_KNOWN_CODES[goodbye.toString()] || ""; this.metrics?.peerGoodbyeSent.inc({reason}); - const conn = getConnection(this.libp2p.connectionManager, peer.toString()); + const conn = getConnection(this.libp2p, peer.toString()); if (conn && Date.now() - conn.stat.timeline.open > LONG_PEER_CONNECTION_MS) { this.metrics?.peerLongConnectionDisconnect.inc({reason}); } @@ -682,7 +687,7 @@ export class PeerManager { peersByClient.set(client, 0); } - for (const connections of getConnectionsMap(this.libp2p.connectionManager).values()) { + for (const connections of getConnectionsMap(this.libp2p).values()) { const openCnx = connections.find((cnx) => cnx.stat.status === "OPEN"); if (openCnx) { const direction = openCnx.stat.direction; diff --git a/packages/beacon-node/src/network/peers/utils/getConnectedPeerIds.ts b/packages/beacon-node/src/network/peers/utils/getConnectedPeerIds.ts index 4aac20f3bddc..11bc7acaee3a 100644 --- a/packages/beacon-node/src/network/peers/utils/getConnectedPeerIds.ts +++ b/packages/beacon-node/src/network/peers/utils/getConnectedPeerIds.ts @@ -8,7 +8,7 @@ import {getConnectionsMap} from "../../util.js"; */ export function getConnectedPeerIds(libp2p: Libp2p): PeerId[] { const peerIds: PeerId[] = []; - for (const connections of getConnectionsMap(libp2p.connectionManager).values()) { + for (const connections of getConnectionsMap(libp2p).values()) { const openConnection = connections.find(isConnectionOpen); if (openConnection) { peerIds.push(openConnection.remotePeer); @@ -21,7 +21,7 @@ export function getConnectedPeerIds(libp2p: Libp2p): PeerId[] { * Efficiently check if there is at least one peer connected */ export function hasSomeConnectedPeer(libp2p: Libp2p): boolean { - for (const connections of getConnectionsMap(libp2p.connectionManager).values()) { + for (const connections of getConnectionsMap(libp2p).values()) { if (connections.some(isConnectionOpen)) { return true; } diff --git a/packages/beacon-node/src/network/util.ts b/packages/beacon-node/src/network/util.ts index ac6660962f81..05d170627f46 100644 --- a/packages/beacon-node/src/network/util.ts +++ b/packages/beacon-node/src/network/util.ts @@ -1,10 +1,7 @@ import type {PeerId} from "@libp2p/interface-peer-id"; import type {Connection} from "@libp2p/interface-connection"; -import type {ConnectionManager} from "@libp2p/interface-connection-manager"; -import type {Components} from "libp2p/components.js"; import type {DefaultConnectionManager} from "libp2p/connection-manager/index.js"; -import type {DefaultDialer} from "libp2p/connection-manager/dialer/index.js"; -import {PeerIdStr} from "../util/peerId.js"; +import type {PeerIdStr} from "../util/peerId.js"; import type {Libp2p} from "./interface.js"; export function prettyPrintPeerId(peerId: PeerId): string { @@ -18,20 +15,19 @@ export function prettyPrintPeerIdStr(id: PeerIdStr): string { /** * Get the connections map from a connection manager */ -// Compat function for type mismatch reasons -export function getConnectionsMap(connectionManager: ConnectionManager): Map { - return (connectionManager as unknown as DefaultConnectionManager)["connections"] as Map; +// Compat function for efficiency reasons +export function getConnectionsMap(libp2p: Libp2p): Map { + return (libp2p.services.components.connectionManager as DefaultConnectionManager)["connections"] as Map< + string, + Connection[] + >; } -export function getConnection(connectionManager: ConnectionManager, peerIdStr: string): Connection | undefined { - return getConnectionsMap(connectionManager).get(peerIdStr)?.[0] ?? undefined; +export function getConnection(libp2p: Libp2p, peerIdStr: string): Connection | undefined { + return getConnectionsMap(libp2p).get(peerIdStr)?.[0] ?? undefined; } // https://github.com/ChainSafe/js-libp2p-gossipsub/blob/3475242ed254f7647798ab7f36b21909f6cb61da/src/index.ts#L2009 export function isPublishToZeroPeersError(e: Error): boolean { return e.message.includes("PublishError.InsufficientPeers"); } - -export function getDefaultDialer(libp2p: Libp2p): DefaultDialer { - return (libp2p as unknown as {components: Components}).components.dialer as DefaultDialer; -} diff --git a/packages/beacon-node/test/e2e/network/gossipsub.test.ts b/packages/beacon-node/test/e2e/network/gossipsub.test.ts index 3e22f3f59191..b943c356ff00 100644 --- a/packages/beacon-node/test/e2e/network/gossipsub.test.ts +++ b/packages/beacon-node/test/e2e/network/gossipsub.test.ts @@ -18,7 +18,7 @@ describe("gossipsub / worker", function () { /* eslint-disable mocha/no-top-level-hooks */ function runTests(this: Mocha.Suite, {useWorker}: {useWorker: boolean}): void { - if (this.timeout() < 15 * 1000) this.timeout(150 * 1000); + if (this.timeout() < 20 * 1000) this.timeout(150 * 1000); this.retries(0); // This test fail sometimes, with a 5% rate. const afterEachCallbacks: (() => Promise | void)[] = []; diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts index 4018af6ea507..aed350a4c76d 100644 --- a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts +++ b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts @@ -2,7 +2,6 @@ import {Connection} from "@libp2p/interface-connection"; import {CustomEvent} from "@libp2p/interfaces/events"; import sinon from "sinon"; import {expect} from "chai"; -import {DefaultConnectionManager} from "libp2p/connection-manager"; import {config} from "@lodestar/config/default"; import {BitArray} from "@chainsafe/ssz"; import {altair, phase0, ssz} from "@lodestar/types"; @@ -155,7 +154,7 @@ describe("network / peers / PeerManager", function () { const {statusCache, libp2p, networkEventBus} = await mockModules(); // Simualate a peer connection, get() should return truthy - getConnectionsMap(libp2p.connectionManager).set(peerId1.toString(), [libp2pConnectionOutboud]); + getConnectionsMap(libp2p).set(peerId1.toString(), [libp2pConnectionOutboud]); // Subscribe to `peerConnected` event, which must fire after checking peer relevance const peerConnectedPromise = waitForEvent(networkEventBus, NetworkEvent.peerConnected, this.timeout() / 2); @@ -174,7 +173,7 @@ describe("network / peers / PeerManager", function () { const {statusCache, libp2p, reqResp, peerManager, networkEventBus} = await mockModules(); // Simualate a peer connection, get() should return truthy - getConnectionsMap(libp2p.connectionManager).set(peerId1.toString(), [libp2pConnectionOutboud]); + getConnectionsMap(libp2p).set(peerId1.toString(), [libp2pConnectionOutboud]); // Subscribe to `peerConnected` event, which must fire after checking peer relevance const peerConnectedPromise = waitForEvent(networkEventBus, NetworkEvent.peerConnected, this.timeout() / 2); @@ -187,9 +186,9 @@ describe("network / peers / PeerManager", function () { reqResp.sendMetadata.resolves(remoteMetadata); // Simualate a peer connection, get() should return truthy - getConnectionsMap(libp2p.connectionManager).set(peerId1.toString(), [libp2pConnectionOutboud]); - (libp2p.connectionManager as DefaultConnectionManager).dispatchEvent( - new CustomEvent("peer:connect", {detail: libp2pConnectionOutboud}) + getConnectionsMap(libp2p).set(peerId1.toString(), [libp2pConnectionOutboud]); + libp2p.services.components.events.dispatchEvent( + new CustomEvent("connection:open", {detail: libp2pConnectionOutboud}) ); await peerConnectedPromise; diff --git a/packages/cli/package.json b/packages/cli/package.json index 1d4d29d5ee9b..5a3873a58316 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -57,7 +57,7 @@ "@chainsafe/bls-keygen": "^0.3.0", "@chainsafe/bls-keystore": "^2.0.0", "@chainsafe/blst": "^0.2.9", - "@chainsafe/discv5": "^3.0.0", + "@chainsafe/discv5": "^4.0.0", "@chainsafe/ssz": "^0.10.2", "@libp2p/peer-id-factory": "^2.0.3", "@lodestar/api": "^1.9.1", @@ -71,7 +71,7 @@ "@lodestar/types": "^1.9.1", "@lodestar/utils": "^1.9.1", "@lodestar/validator": "^1.9.1", - "@multiformats/multiaddr": "^11.0.0", + "@multiformats/multiaddr": "^12.1.3", "@types/lockfile": "^1.0.2", "bip39": "^3.1.0", "deepmerge": "^4.3.1", diff --git a/packages/db/package.json b/packages/db/package.json index ccabf40e3efc..30559b5b3775 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -41,7 +41,7 @@ "@lodestar/config": "^1.9.1", "@lodestar/utils": "^1.9.1", "@types/levelup": "^4.3.3", - "it-all": "^3.0.1", + "it-all": "^3.0.2", "level": "^8.0.0" }, "devDependencies": { diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 50e65f79e2ef..b36c791acbe8 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -55,12 +55,12 @@ }, "dependencies": { "@chainsafe/fast-crc32c": "^4.1.1", - "@libp2p/interface-connection": "^3.0.2", - "@libp2p/interface-peer-id": "^2.0.1", + "@libp2p/interface-connection": "^5.1.0", + "@libp2p/interface-peer-id": "^2.0.2", "@lodestar/config": "^1.9.1", "@lodestar/params": "^1.9.1", "@lodestar/utils": "^1.9.1", - "it-all": "^3.0.1", + "it-all": "^3.0.2", "snappy": "^7.2.2", "snappyjs": "^0.7.0", "uint8arraylist": "^2.4.3", @@ -69,10 +69,10 @@ "devDependencies": { "@lodestar/logger": "^1.9.1", "@lodestar/types": "^1.9.1", - "libp2p": "0.42.2" + "libp2p": "0.45.9" }, "peerDependencies": { - "libp2p": "~0.42.2" + "libp2p": "~0.45.0" }, "keywords": [ "ethereum", diff --git a/packages/reqresp/src/encoders/requestDecode.ts b/packages/reqresp/src/encoders/requestDecode.ts index c29317d8664c..4095e238196d 100644 --- a/packages/reqresp/src/encoders/requestDecode.ts +++ b/packages/reqresp/src/encoders/requestDecode.ts @@ -12,7 +12,9 @@ const EMPTY_DATA = new Uint8Array(); * request ::= | * ``` */ -export function requestDecode(protocol: MixedProtocol): Sink> { +export function requestDecode( + protocol: MixedProtocol +): Sink, Promise> { return async function requestDecodeSink(source) { const type = protocol.requestSizes; if (type === null) { diff --git a/packages/reqresp/test/unit/response/index.test.ts b/packages/reqresp/test/unit/response/index.test.ts index e712f6b0ff8c..28d6a71b54b0 100644 --- a/packages/reqresp/test/unit/response/index.test.ts +++ b/packages/reqresp/test/unit/response/index.test.ts @@ -56,7 +56,7 @@ describe("response / handleRequest", () => { for (const {id, requestChunks, protocol, expectedResponseChunks, expectedError} of testCases) { it(id, async () => { - const stream = new MockLibP2pStream(requestChunks); + const stream = new MockLibP2pStream(requestChunks as any); const rateLimiter = new ReqRespRateLimiter({rateLimitMultiplier: 0}); const resultPromise = handleRequest({ diff --git a/packages/reqresp/test/utils/index.ts b/packages/reqresp/test/utils/index.ts index 2ba3afd25976..aae6e5fdcf3c 100644 --- a/packages/reqresp/test/utils/index.ts +++ b/packages/reqresp/test/utils/index.ts @@ -1,5 +1,6 @@ import {Stream, StreamStat} from "@libp2p/interface-connection"; import {expect} from "chai"; +import {Uint8ArrayList} from "uint8arraylist"; import {toHexString} from "@chainsafe/ssz"; import {fromHex} from "@lodestar/utils"; import {ResponseIncoming, RespStatus} from "../../src/index.js"; @@ -45,8 +46,10 @@ export class MockLibP2pStream implements Stream { source: Stream["source"]; resultChunks: Uint8Array[] = []; - constructor(requestChunks: Uint8Array[] | AsyncIterable | AsyncGenerator, protocol?: string) { - this.source = Array.isArray(requestChunks) ? arrToSource(requestChunks) : requestChunks; + constructor(requestChunks: Uint8ArrayList[] | AsyncIterable | AsyncGenerator, protocol?: string) { + this.source = Array.isArray(requestChunks) + ? arrToSource(requestChunks) + : (requestChunks as AsyncGenerator); this.stat.protocol = protocol ?? "mock"; } diff --git a/types/it-pipe/index.d.ts b/types/it-pipe/index.d.ts index ecc29a856464..0abe013aad78 100644 --- a/types/it-pipe/index.d.ts +++ b/types/it-pipe/index.d.ts @@ -7,10 +7,10 @@ declare module "it-pipe" { type Source = AsyncIterable; - type Sink = (source: Source) => TReturn; - type Duplex = { - sink: Sink; - source: Source; + type Sink = (source: TSource) => TReturn; + type Duplex = { + sink: Sink; + source: TSource; }; export function pipe(f1: A | (() => A), f2: (source: A) => B): B; diff --git a/yarn.lock b/yarn.lock index abf9763d38da..6f8d87567688 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10,19 +10,19 @@ jsbn "1.1.0" sprintf-js "1.1.2" -"@achingbrain/nat-port-mapper@^1.0.3": - version "1.0.7" - resolved "https://registry.yarnpkg.com/@achingbrain/nat-port-mapper/-/nat-port-mapper-1.0.7.tgz#82c414712da38a0f3da0f938982b6dd724d3c677" - integrity sha512-P8Z8iMZBQCsN7q3XoVoJAX3CGPUTbGTh1XBU8JytCW3hBmSk594l8YvdrtY5NVexVHSwLeiXnDsP4d10NJHaeg== +"@achingbrain/nat-port-mapper@^1.0.9": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@achingbrain/nat-port-mapper/-/nat-port-mapper-1.0.9.tgz#8e61cf6f5dbeaa55c4e64a0023a362d4a1f61a36" + integrity sha512-w1M7dh7IsO5fvX9VQpH0w8MMphzLUl52Kf+paXTScNmFH4Ua+R6XI+x5p7LI3vY36JkTllTqAxNo8g1y0CMCrA== dependencies: "@achingbrain/ssdp" "^4.0.1" "@libp2p/logger" "^2.0.0" default-gateway "^6.0.2" err-code "^3.0.1" - it-first "^1.0.7" + it-first "^3.0.1" p-defer "^4.0.0" - p-timeout "^5.0.2" - xml2js "^0.4.23" + p-timeout "^6.1.1" + xml2js "^0.6.0" "@achingbrain/ssdp@^4.0.1": version "4.0.1" @@ -331,7 +331,12 @@ dependencies: "@babel/types" "^7.11.0" -"@babel/helper-validator-identifier@^7.10.4", "@babel/helper-validator-identifier@^7.14.9": +"@babel/helper-string-parser@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" + integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== + +"@babel/helper-validator-identifier@^7.10.4": version "7.14.9" resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.9.tgz" integrity sha512-pQYxPY0UP6IHISRitNe8bsijHex4TWZXi2HwKVsjPiltzlhse2znVcm9Ace510VT1kxIHjGJCZZQBX2gJDbo0g== @@ -341,6 +346,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2" integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w== +"@babel/helper-validator-identifier@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193" + integrity sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ== + "@babel/helpers@^7.10.4": version "7.10.4" resolved "https://registry.npmjs.org/@babel/helpers/-/helpers-7.10.4.tgz" @@ -397,21 +407,13 @@ globals "^11.1.0" lodash "^4.17.19" -"@babel/types@^7.10.4", "@babel/types@^7.11.0": - version "7.11.0" - resolved "https://registry.npmjs.org/@babel/types/-/types-7.11.0.tgz" - integrity sha512-O53yME4ZZI0jO1EVGtF1ePGl0LHirG4P1ibcD80XyzZcKhcMFeCXmh4Xb1ifGBIV233Qg12x4rBfQgA+tmOukA== - dependencies: - "@babel/helper-validator-identifier" "^7.10.4" - lodash "^4.17.19" - to-fast-properties "^2.0.0" - -"@babel/types@^7.15.0": - version "7.15.0" - resolved "https://registry.npmjs.org/@babel/types/-/types-7.15.0.tgz" - integrity sha512-OBvfqnllOIdX4ojTHpwZbpvz4j3EWyjkZEdmjH0/cgsd6QOdSgU8rLSk6ard/pcW7rlmjdVSX/AWOaORR1uNOQ== +"@babel/types@^7.10.4", "@babel/types@^7.11.0", "@babel/types@^7.15.0": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.5.tgz#cd93eeaab025880a3a47ec881f4b096a5b786fbe" + integrity sha512-zo3MIHGOkPOfoRXitsgHLjEXmlDaD/5KU1Uzuc9GNiZPhSqVxVRtxuPaSBZDsYZ9qV88AjtMtWW7ww98loJ9KA== dependencies: - "@babel/helper-validator-identifier" "^7.14.9" + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.5" to-fast-properties "^2.0.0" "@balena/dockerignore@^1.0.2": @@ -453,7 +455,7 @@ "@chainsafe/bls-keygen@^0.3.0": version "0.3.0" - resolved "https://registry.npmjs.org/@chainsafe/bls-keygen/-/bls-keygen-0.3.0.tgz" + resolved "https://registry.yarnpkg.com/@chainsafe/bls-keygen/-/bls-keygen-0.3.0.tgz#d7472a945f6f49b5cb357241bfba2f5c12a635c5" integrity sha512-5Iq6E5E987hyio74G1fXPYI3t9iVeHxRX1tDMpnCV9T82rPz061yFsMz3W3aXE26+k6+fcz0bsYX3ijOizkx+A== dependencies: "@chainsafe/bls-hd-key" "^0.2.0" @@ -498,18 +500,18 @@ node-fetch "^2.6.1" node-gyp "^8.4.0" -"@chainsafe/discv5@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@chainsafe/discv5/-/discv5-3.0.0.tgz#ee9c7d1631c95382ee381831c2c87c5cf1dd75c2" - integrity sha512-NqRjJ9tfD8bbxCo6oI2y+Ih1fBlHmIs8l19f5ca5lY61nfISLKMUO+uCvJ6mguqcSzz0zxSsp5dmzhJ0E+J+HA== +"@chainsafe/discv5@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@chainsafe/discv5/-/discv5-4.0.0.tgz#41b9876ce0d209a4abcf844cfcdb6c4c8c338fe5" + integrity sha512-4dsfSAAa2rSRgrYM7YdvJRYNMSZ0NZzzBJw7e+PQMURvCZBRtDxh51/WS0qFX2sFrmjkT+i+xmLD8EsS4P5pUw== dependencies: "@libp2p/crypto" "^1.0.0" - "@libp2p/interface-peer-discovery" "^1.0.1" + "@libp2p/interface-peer-discovery" "^2.0.0" "@libp2p/interface-peer-id" "^2.0.1" "@libp2p/interface-peer-info" "^1.0.1" "@libp2p/interfaces" "^3.0.2" "@libp2p/peer-id" "^2.0.1" - "@multiformats/multiaddr" "^11.0.0" + "@multiformats/multiaddr" "^12.1.3" base64url "^3.0.1" bcrypto "^5.4.0" bigint-buffer "^1.1.5" @@ -545,57 +547,56 @@ resolved "https://registry.yarnpkg.com/@chainsafe/is-ip/-/is-ip-2.0.1.tgz#62cb285669d91f88fd9fa285048dde3882f0993b" integrity sha512-nqSJ8u2a1Rv9FYbyI8qpDhTYujaKEyLknNrTejLYoSWmdeg+2WB7R6BZqPZYfrJzDxVi3rl6ZQuoaEvpKRZWgQ== -"@chainsafe/libp2p-gossipsub@^6.2.0": - version "6.2.0" - resolved "https://registry.yarnpkg.com/@chainsafe/libp2p-gossipsub/-/libp2p-gossipsub-6.2.0.tgz#1266ae5a10cd57e297bd30edf3b365c907ce78e7" - integrity sha512-b3xEgjaatCmzJgNyE7qbTl/JBIymcNWbLUtW1nGA9a0n9Y0IjnNLyUmHH0y3xe22trVTAf6o7qpAdkbXILU9sg== +"@chainsafe/libp2p-gossipsub@^9.1.0": + version "9.1.0" + resolved "https://registry.yarnpkg.com/@chainsafe/libp2p-gossipsub/-/libp2p-gossipsub-9.1.0.tgz#0aee7960426e323f1da5774e16742b0ba9a015e6" + integrity sha512-zc1Jx0DcVNH0iAncDlyd0/rAN9mWCpIxrmrw8eC6/gvHDmi24y8orDH8Npj2Naydh/ukeDhG9Iqx4Dnoe8V51w== dependencies: "@libp2p/crypto" "^1.0.3" - "@libp2p/interface-connection" "^3.0.1" - "@libp2p/interface-connection-manager" "^1.3.0" + "@libp2p/interface-connection" "^5.0.1" + "@libp2p/interface-connection-manager" "^3.0.1" "@libp2p/interface-keys" "^1.0.3" "@libp2p/interface-peer-id" "^2.0.0" - "@libp2p/interface-peer-store" "^1.2.2" - "@libp2p/interface-pubsub" "^3.0.0" + "@libp2p/interface-peer-store" "^2.0.3" + "@libp2p/interface-pubsub" "^4.0.0" "@libp2p/interface-registrar" "^2.0.3" "@libp2p/interfaces" "^3.2.0" "@libp2p/logger" "^2.0.0" "@libp2p/peer-id" "^2.0.0" "@libp2p/peer-record" "^5.0.0" - "@libp2p/pubsub" "^6.0.0" + "@libp2p/pubsub" "^7.0.1" "@libp2p/topology" "^4.0.0" - "@multiformats/multiaddr" "^11.0.0" - abortable-iterator "^4.0.2" + "@multiformats/multiaddr" "^12.0.0" + abortable-iterator "^5.0.1" denque "^1.5.0" - it-length-prefixed "^8.0.2" - it-pipe "^2.0.4" + it-length-prefixed "^9.0.1" + it-pipe "^3.0.1" it-pushable "^3.1.0" multiformats "^11.0.0" protobufjs "^6.11.2" uint8arraylist "^2.3.2" uint8arrays "^4.0.2" -"@chainsafe/libp2p-noise@^11.0.4": - version "11.0.4" - resolved "https://registry.yarnpkg.com/@chainsafe/libp2p-noise/-/libp2p-noise-11.0.4.tgz#b4806e7605e44fa279130c60a95faad13ed01d93" - integrity sha512-X7kA6a3/QPFxNFwgUJ8vubDu5qBDcDT0nhD+jL7g60IFKZu//HFH7oqsNCZa12yx0oR1fEYOR62iHDt2GHyWBQ== +"@chainsafe/libp2p-noise@^12.0.1": + version "12.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/libp2p-noise/-/libp2p-noise-12.0.1.tgz#140a4c2e6976fe60e6ccb391a9493b83a28430dc" + integrity sha512-VYuc5a3raIcCmv4F+LOfez7/9rmMgfjNo9h66cspLJKHuWgpzzIRRL9srVth6VC5DMjftExHM0aZv47Tf5govQ== dependencies: "@libp2p/crypto" "^1.0.11" - "@libp2p/interface-connection-encrypter" "^3.0.5" + "@libp2p/interface-connection-encrypter" "^4.0.0" "@libp2p/interface-keys" "^1.0.6" "@libp2p/interface-metrics" "^4.0.4" "@libp2p/interface-peer-id" "^2.0.0" "@libp2p/logger" "^2.0.5" "@libp2p/peer-id" "^2.0.0" + "@noble/hashes" "^1.3.0" "@stablelib/chacha20poly1305" "^1.0.1" - "@stablelib/hkdf" "^1.0.1" - "@stablelib/sha256" "^1.0.1" "@stablelib/x25519" "^1.0.3" - it-length-prefixed "^8.0.2" + it-length-prefixed "^9.0.1" it-pair "^2.0.2" - it-pb-stream "^3.2.0" - it-pipe "^2.0.3" - it-stream-types "^1.0.4" + it-pb-stream "^4.0.1" + it-pipe "^3.0.1" + it-stream-types "^2.0.1" protons-runtime "^5.0.0" uint8arraylist "^2.3.2" uint8arrays "^4.0.2" @@ -1708,6 +1709,11 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" +"@leichtgewicht/ip-codec@^2.0.1": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz#b2ac626d6cb9c8718ab459166d4bb405b8ffa78b" + integrity sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A== + "@lerna/child-process@6.6.1": version "6.6.1" resolved "https://registry.yarnpkg.com/@lerna/child-process/-/child-process-6.6.1.tgz#e31bc411ad6d474cf7b676904da6f77f58fd64eb" @@ -1804,339 +1810,315 @@ write-pkg "4.0.0" yargs "16.2.0" -"@libp2p/bootstrap@^6.0.3": - version "6.0.3" - resolved "https://registry.yarnpkg.com/@libp2p/bootstrap/-/bootstrap-6.0.3.tgz#0e91542808972ac966919d2b0a5bcdbf71144ca7" - integrity sha512-0/pDxBn8+rLtZfGX2PHzOVT3wBATOv4SPiKWjHMeiSfIWQI3kQ0bZDgLp+2lnG8j1JVGDtYJVpmYTpEzlVgbRA== +"@libp2p/bootstrap@^8.0.0": + version "8.0.0" + resolved "https://registry.yarnpkg.com/@libp2p/bootstrap/-/bootstrap-8.0.0.tgz#dc9fc3f0367953b69c7b726a205ba5d17a8eac50" + integrity sha512-xbaJ+ybx1FGsi8FeGl9g1Wk6P2zf5/Thdk9Fe1qXV0O0xIW0xRWrefOYG5Dvt+BV54C/zlnQ4CG+Xs+Rr7wsbA== dependencies: - "@libp2p/interface-peer-discovery" "^1.0.1" + "@libp2p/interface-peer-discovery" "^2.0.0" "@libp2p/interface-peer-info" "^1.0.7" - "@libp2p/interface-peer-store" "^1.2.2" + "@libp2p/interface-peer-store" "^2.0.0" "@libp2p/interfaces" "^3.0.3" "@libp2p/logger" "^2.0.1" "@libp2p/peer-id" "^2.0.0" "@multiformats/mafmt" "^12.0.0" "@multiformats/multiaddr" "^12.0.0" -"@libp2p/crypto@^1.0.0", "@libp2p/crypto@^1.0.11", "@libp2p/crypto@^1.0.3", "@libp2p/crypto@^1.0.4": - version "1.0.11" - resolved "https://registry.yarnpkg.com/@libp2p/crypto/-/crypto-1.0.11.tgz#c930c64abb189654cf8294d36fe9c23a62ceb4ea" - integrity sha512-DWiG/0fKIDnkhTF3HoCu2OzkuKXysR/UKGdM9JZkT6F9jS9rwZYEwmacs4ybw1qyufyH+pMXV3/vuUu2Q/UxLw== +"@libp2p/crypto@^1.0.0", "@libp2p/crypto@^1.0.11", "@libp2p/crypto@^1.0.17", "@libp2p/crypto@^1.0.3": + version "1.0.17" + resolved "https://registry.yarnpkg.com/@libp2p/crypto/-/crypto-1.0.17.tgz#e64043328c0c866bf7f4cc8560b4f483e9c745dc" + integrity sha512-Oeg0Eb/EvAho0gVkOgemXEgrVxWaT3x/DpFgkBdZ9qGxwq75w/E/oPc7souqBz+l1swfz37GWnwV7bIb4Xv5Ag== dependencies: "@libp2p/interface-keys" "^1.0.2" + "@libp2p/interfaces" "^3.2.0" "@noble/ed25519" "^1.6.0" "@noble/secp256k1" "^1.5.4" - err-code "^3.0.1" multiformats "^11.0.0" node-forge "^1.1.0" - protons-runtime "^4.0.1" + protons-runtime "^5.0.0" + uint8arraylist "^2.4.3" uint8arrays "^4.0.2" -"@libp2p/interface-address-manager@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@libp2p/interface-address-manager/-/interface-address-manager-2.0.0.tgz#3088fe5319bc39351a11dadae025f2865ebad991" - integrity sha512-cEzZMxgy71geUcNMZtbF/gNNZbtc8Gx6MI/bj2sPT7ZVqMR7XhSCrpzm3kBkWpSHdWMYImfXCwU0oLg4UtI9Ow== +"@libp2p/interface-address-manager@^3.0.0": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@libp2p/interface-address-manager/-/interface-address-manager-3.0.1.tgz#050701e904c03ee5f2c974b734cbaab1f5b7ac59" + integrity sha512-8N1nfOtZ/CnZ/cL0Bnj59fhcSs7orI4evmNVsv2DM1VaNHXqc9tPy8JmQE2HRjrUXeUPwtzzG2eoP7l0ZYdC0g== dependencies: - "@libp2p/interfaces" "^3.0.0" - "@multiformats/multiaddr" "^11.0.0" + "@multiformats/multiaddr" "^12.0.0" -"@libp2p/interface-connection-encrypter@^3.0.1", "@libp2p/interface-connection-encrypter@^3.0.5": - version "3.0.6" - resolved "https://registry.yarnpkg.com/@libp2p/interface-connection-encrypter/-/interface-connection-encrypter-3.0.6.tgz#1f7c7428d5905b390cfc5390e72bd02829213d31" - integrity sha512-LwyYBN/aSa3IPCe7gBxffx/vaC0rFxAXlCbx4QGaWGtg6qK80Ouj89LEDWb3HkMbecNVWaV4TEqJIM5WnAAx1Q== +"@libp2p/interface-connection-encrypter@^4.0.0": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@libp2p/interface-connection-encrypter/-/interface-connection-encrypter-4.0.1.tgz#8eea0889fb5a7bafaf331ab0e6f2d341e0734c56" + integrity sha512-fOtZpaFL2f5vID/RaBpVMAR9OKx5DmDT/yMEFTCarNc6Bb37fWwClI4WNCtoVbDQwcnr4H4ZIo0+9yCxjEIjjQ== dependencies: "@libp2p/interface-peer-id" "^2.0.0" - it-stream-types "^1.0.4" - uint8arraylist "^2.1.2" + it-stream-types "^2.0.1" -"@libp2p/interface-connection-manager@^1.1.1", "@libp2p/interface-connection-manager@^1.3.0": - version "1.3.7" - resolved "https://registry.yarnpkg.com/@libp2p/interface-connection-manager/-/interface-connection-manager-1.3.7.tgz#110a3ea0a8e63461e159df7182e6246625e92bd5" - integrity sha512-GyRa7FXtwjbch4ucFa/jj6vcaQT2RyhUbH3q0tIOTzjntABTMzQrhn3BWOGU5deRp2K7cVOB/OzrdhHdGUxYQA== +"@libp2p/interface-connection-gater@^3.0.0": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@libp2p/interface-connection-gater/-/interface-connection-gater-3.0.1.tgz#073080a7703d7525e0ea6d64bb3f951e0a7728a9" + integrity sha512-3a+EmcKFIdYVM6tmmIKZt/4fREPApA/Z/PZHOEa4lqJA9c/BHO1HTq0YzEoYsptudYTcdhQLgpYzh8FVhfZGDg== dependencies: - "@libp2p/interface-connection" "^3.0.0" + "@libp2p/interface-connection" "^5.0.0" "@libp2p/interface-peer-id" "^2.0.0" - "@libp2p/interfaces" "^3.0.0" - "@multiformats/multiaddr" "^11.0.0" + "@multiformats/multiaddr" "^12.0.0" -"@libp2p/interface-connection@^3.0.0", "@libp2p/interface-connection@^3.0.1", "@libp2p/interface-connection@^3.0.2": - version "3.0.8" - resolved "https://registry.yarnpkg.com/@libp2p/interface-connection/-/interface-connection-3.0.8.tgz#2c17bcdc53c6951d96a8430bb7dad1cb064cf184" - integrity sha512-JiI9xVPkiSgW9hkvHWA4e599OLPNSACrpgtx6UffHG9N+Jpt0IOmM4iLic8bSIYkZJBOQFG1Sv/gVNB98Uq0Nw== +"@libp2p/interface-connection-manager@^3.0.0", "@libp2p/interface-connection-manager@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@libp2p/interface-connection-manager/-/interface-connection-manager-3.0.1.tgz#2181e28e62f15e33323a293147f3da85b537939d" + integrity sha512-7ZAvzOWfHs3BtaoZoWsT+Ks1bo6HjyRMq1SJdFWDJ+ZkYEzrf6sdtQwsX8eXhwRDO6PuzpUDqLZ9TNQ2GVKEEw== dependencies: + "@libp2p/interface-connection" "^5.0.0" "@libp2p/interface-peer-id" "^2.0.0" "@libp2p/interfaces" "^3.0.0" - "@multiformats/multiaddr" "^11.0.0" - it-stream-types "^1.0.4" - uint8arraylist "^2.1.2" + "@libp2p/peer-collections" "^3.0.1" + "@multiformats/multiaddr" "^12.0.0" -"@libp2p/interface-content-routing@^2.0.0": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@libp2p/interface-content-routing/-/interface-content-routing-2.0.1.tgz#e050dc42adc3e9b4f1666eafa889c88f892ba1c4" - integrity sha512-M3rYXMhH+102qyZzc0GzkKq10x100nWVXGns2qtN3O82Hy/6FxXdgLUGIGWMdCj/7ilaVAuTwx8V3+DGmDIiMw== +"@libp2p/interface-connection@^5.0.0", "@libp2p/interface-connection@^5.0.1", "@libp2p/interface-connection@^5.0.2", "@libp2p/interface-connection@^5.1.0", "@libp2p/interface-connection@^5.1.1": + version "5.1.1" + resolved "https://registry.yarnpkg.com/@libp2p/interface-connection/-/interface-connection-5.1.1.tgz#da0572c76da43629d52b8bec6cd092143fae421d" + integrity sha512-ytknMbuuNW72LYMmTP7wFGP5ZTaUSGBCmV9f+uQ55XPcFHtKXLtKWVU/HE8IqPmwtyU8AO7veGoJ/qStMHNRVA== dependencies: - "@libp2p/interface-peer-info" "^1.0.0" + "@libp2p/interface-peer-id" "^2.0.0" "@libp2p/interfaces" "^3.0.0" - multiformats "^11.0.0" + "@multiformats/multiaddr" "^12.0.0" + it-stream-types "^2.0.1" + uint8arraylist "^2.4.3" -"@libp2p/interface-dht@^2.0.0": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@libp2p/interface-dht/-/interface-dht-2.0.1.tgz#b41901d193081b6e51a2dd55e7338ed03a2bdd07" - integrity sha512-+yEbt+1hMTR1bITzYyE771jEujimPXqDyFm8T1a8slMpeOD9z5wmLfuCWif8oGZJzXX5YqldWwSwytJQgWXL9g== +"@libp2p/interface-content-routing@^2.0.0", "@libp2p/interface-content-routing@^2.1.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@libp2p/interface-content-routing/-/interface-content-routing-2.1.1.tgz#7c56acad48f59feb9f0c6dd637e73d0e4eebd510" + integrity sha512-nRPOUWgq1K1fDr3FKW93Tip7aH8AFefCw3nJygL4crepxWTSGw95s1GyDpC7t0RJkWTRNHsqZvsFsJ9FkHExKw== dependencies: - "@libp2p/interface-peer-discovery" "^1.0.0" - "@libp2p/interface-peer-id" "^2.0.0" "@libp2p/interface-peer-info" "^1.0.0" "@libp2p/interfaces" "^3.0.0" multiformats "^11.0.0" -"@libp2p/interface-keychain@^2.0.0": - version "2.0.3" - resolved "https://registry.yarnpkg.com/@libp2p/interface-keychain/-/interface-keychain-2.0.3.tgz#3cbdb251b9e9c496976d337a62d2e6d5b7415035" - integrity sha512-qtSUww/lpnrDHYMAOGDz5KLuTrHNM15kyuLqop96uN22V7PDizvkHY4EgtqWKgPLoNyeEnMwfUSBOQbXcWuVUA== +"@libp2p/interface-keychain@^2.0.0", "@libp2p/interface-keychain@^2.0.3", "@libp2p/interface-keychain@^2.0.4": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@libp2p/interface-keychain/-/interface-keychain-2.0.5.tgz#6ce104f38cf07ad72c9dfbe471a689f4ea4b4687" + integrity sha512-mb7QNgn9fIvC7CaJCi06GJ+a6DN6RVT9TmEi0NmedZGATeCArPeWWG7r7IfxNVXb9cVOOE1RzV1swK0ZxEJF9Q== dependencies: "@libp2p/interface-peer-id" "^2.0.0" multiformats "^11.0.0" -"@libp2p/interface-keys@^1.0.2", "@libp2p/interface-keys@^1.0.3": - version "1.0.3" - resolved "https://registry.yarnpkg.com/@libp2p/interface-keys/-/interface-keys-1.0.3.tgz#251abb2f0fe084e35e16ba782d64c7e4dfb24470" - integrity sha512-K8/HlRl/swbVTWuGHNHF28EytszYfUhKgUHfv8CdbMk9ZA/bgO4uU+d9rcrg/Dhw3511U3aRz2bwl2psn6rJfg== - -"@libp2p/interface-keys@^1.0.6": - version "1.0.7" - resolved "https://registry.yarnpkg.com/@libp2p/interface-keys/-/interface-keys-1.0.7.tgz#ad09ee7dc9c1495f1dd3e1785133c317befb4d7b" - integrity sha512-DRMPY9LfcnGJKrjaqIkY62U3fW2dya3VLy4x986ExtMrGn4kxIHeQ1IKk8/Vs9CJHTKmXEMID4of1Cjnw4aJpA== +"@libp2p/interface-keys@^1.0.2", "@libp2p/interface-keys@^1.0.3", "@libp2p/interface-keys@^1.0.6": + version "1.0.8" + resolved "https://registry.yarnpkg.com/@libp2p/interface-keys/-/interface-keys-1.0.8.tgz#2c6b55136113ae7cf78133d3c459cdf0455b29ae" + integrity sha512-CJ1SlrwuoHMquhEEWS77E+4vv7hwB7XORkqzGQrPQmA9MRdIEZRS64bA4JqCLUDa4ltH0l+U1vp0oZHLT67NEA== -"@libp2p/interface-libp2p@^1.0.0": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@libp2p/interface-libp2p/-/interface-libp2p-1.1.1.tgz#0e75af940dcc0f48c6abd677902d3eafc69ac7e8" - integrity sha512-cELZZv/tzFxbUzL3Jvbk+AM2J6kDhIUNBIMMMLuR3LIHfmVJkh31G3ChLUZuKhBwB8wXJ1Ssev3fk1tfz/5DWA== +"@libp2p/interface-libp2p@^3.1.0", "@libp2p/interface-libp2p@^3.2.0": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@libp2p/interface-libp2p/-/interface-libp2p-3.2.0.tgz#875df729edcb43aee7f8b91191b7fc16d83cb912" + integrity sha512-Vow6xNdjpQ0M/Kt3EDz1qE/Os5OZUyhFt0YTPU5Fp3/kXw/6ocsxYq/Bzird/96gjUjU5/i+Vukn4WgctJf55Q== dependencies: - "@libp2p/interface-connection" "^3.0.0" + "@libp2p/interface-connection" "^5.0.0" "@libp2p/interface-content-routing" "^2.0.0" - "@libp2p/interface-dht" "^2.0.0" "@libp2p/interface-keychain" "^2.0.0" "@libp2p/interface-metrics" "^4.0.0" "@libp2p/interface-peer-id" "^2.0.0" "@libp2p/interface-peer-info" "^1.0.0" "@libp2p/interface-peer-routing" "^1.0.0" - "@libp2p/interface-peer-store" "^1.0.0" - "@libp2p/interface-pubsub" "^3.0.0" + "@libp2p/interface-peer-store" "^2.0.0" "@libp2p/interface-registrar" "^2.0.0" + "@libp2p/interface-transport" "^4.0.0" "@libp2p/interfaces" "^3.0.0" - "@multiformats/multiaddr" "^11.0.0" - -"@libp2p/interface-metrics@^4.0.0", "@libp2p/interface-metrics@^4.0.2": - version "4.0.2" - resolved "https://registry.yarnpkg.com/@libp2p/interface-metrics/-/interface-metrics-4.0.2.tgz#329a1602f7844f6a9cf441439001f8e8f8e7dafc" - integrity sha512-HON9yXhFaTnQ86tOdE18bFJv71zQdI7xrZJuA6pNUtpsfA+djhqWXv0a4mwEGUP7k4zz3FkH0M9CrrvL0pkBWg== - dependencies: - "@libp2p/interface-connection" "^3.0.0" + "@multiformats/multiaddr" "^12.0.0" -"@libp2p/interface-metrics@^4.0.4": - version "4.0.5" - resolved "https://registry.yarnpkg.com/@libp2p/interface-metrics/-/interface-metrics-4.0.5.tgz#92af389705bded1fd3ed7979768cf7a0f7b13b47" - integrity sha512-srBeky1ugu1Bzw9VHGg8ta15oLh+P2PEIsg0cI9qzDbtCJaWGq/IIetpfuaJNVOuBD1CGEEbITNmsk4tDwIE0w== +"@libp2p/interface-metrics@^4.0.0", "@libp2p/interface-metrics@^4.0.2", "@libp2p/interface-metrics@^4.0.4": + version "4.0.8" + resolved "https://registry.yarnpkg.com/@libp2p/interface-metrics/-/interface-metrics-4.0.8.tgz#06eb45588737d72f074c70df8d1ef067a2d7cf71" + integrity sha512-1b9HjYyJH0m35kvPHipuoz2EtYCxyq34NUhuV8VK1VNtrouMpA3uCKp5FI7yHCA6V6+ux1R3UriKgNFOSGbIXQ== dependencies: - "@libp2p/interface-connection" "^3.0.0" + "@libp2p/interface-connection" "^5.0.0" -"@libp2p/interface-peer-discovery@^1.0.0", "@libp2p/interface-peer-discovery@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@libp2p/interface-peer-discovery/-/interface-peer-discovery-1.0.1.tgz#56d14a933a479e9866b1eb41a597717d2e7d954e" - integrity sha512-ZqBhpX7fR3ROYQaGYV47YhyTJJzFDzyyEIsQ7NnDuG3KhcQb2PtocnN0sy1Ozm784M0oYveM/HjfuNxxcOwdYg== +"@libp2p/interface-peer-discovery@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@libp2p/interface-peer-discovery/-/interface-peer-discovery-2.0.0.tgz#90f176cfd202f5a362912386199e64f8b1e0fc53" + integrity sha512-Mien5t3Tc+ntP5p50acKUYJN90ouMnq1lOTQDKQNvGcXoajG8A1AEYLocnzVia/MXiexuj6S/Q28WBBacoOlBg== dependencies: "@libp2p/interface-peer-info" "^1.0.0" "@libp2p/interfaces" "^3.0.0" -"@libp2p/interface-peer-id@^2.0.0", "@libp2p/interface-peer-id@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@libp2p/interface-peer-id/-/interface-peer-id-2.0.1.tgz#445632909d44a8ae2c736bb2aa98c8bf757e8c62" - integrity sha512-k01hKHTAZWMOiBC+yyFsmBguEMvhPkXnQtqLtFqga2fVZu8Zve7zFAtQYLhQjeJ4/apeFtO6ddTS8mCE6hl4OA== +"@libp2p/interface-peer-id@^2.0.0", "@libp2p/interface-peer-id@^2.0.1", "@libp2p/interface-peer-id@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@libp2p/interface-peer-id/-/interface-peer-id-2.0.2.tgz#6302e70b6fc17c451bc3daa11447d059357bcc32" + integrity sha512-9pZp9zhTDoVwzRmp0Wtxw0Yfa//Yc0GqBCJi3EznBDE6HGIAVvppR91wSh2knt/0eYg0AQj7Y35VSesUTzMCUg== dependencies: multiformats "^11.0.0" -"@libp2p/interface-peer-info@^1.0.0", "@libp2p/interface-peer-info@^1.0.1", "@libp2p/interface-peer-info@^1.0.3", "@libp2p/interface-peer-info@^1.0.7": - version "1.0.8" - resolved "https://registry.yarnpkg.com/@libp2p/interface-peer-info/-/interface-peer-info-1.0.8.tgz#8380e9e40d0ec2c8be8e1a43e8a82ae97a0687c4" - integrity sha512-LRvZt/9bZFYW7seAwuSg2hZuPl+FRTAsij5HtyvVwmpfVxipm6yQrKjQ+LiK/SZhIDVsSJ+UjF0mluJj+jeAzQ== +"@libp2p/interface-peer-info@^1.0.0", "@libp2p/interface-peer-info@^1.0.1", "@libp2p/interface-peer-info@^1.0.3", "@libp2p/interface-peer-info@^1.0.7", "@libp2p/interface-peer-info@^1.0.8": + version "1.0.10" + resolved "https://registry.yarnpkg.com/@libp2p/interface-peer-info/-/interface-peer-info-1.0.10.tgz#566026de95a0817b9e853c982b313541b7960c0b" + integrity sha512-HQlo8NwQjMyamCHJrnILEZz+YwEOXCB2sIIw3slIrhVUYeYlTaia1R6d9umaAeLHa255Zmdm4qGH8rJLRqhCcg== dependencies: "@libp2p/interface-peer-id" "^2.0.0" - "@multiformats/multiaddr" "^11.0.0" + "@multiformats/multiaddr" "^12.0.0" -"@libp2p/interface-peer-routing@^1.0.0", "@libp2p/interface-peer-routing@^1.0.1": - version "1.0.7" - resolved "https://registry.yarnpkg.com/@libp2p/interface-peer-routing/-/interface-peer-routing-1.0.7.tgz#043a3341ecb640f6ee36fe600788f7fdcce5bfd0" - integrity sha512-0zxOOmKD6nA3LaArcP9UdRO4vJzEyoRtE34vvQP41UxjcSTaj4em5Fl4Q0RuOMXYPtRp+LdXRYbjJgCSeQoxwA== +"@libp2p/interface-peer-routing@^1.0.0", "@libp2p/interface-peer-routing@^1.1.0": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@libp2p/interface-peer-routing/-/interface-peer-routing-1.1.1.tgz#b4d3f51d996ce0ea19773db45aff4684e247e6fb" + integrity sha512-/XEhwob9qXjdmI8PBcc+qFin32xmtyoC58nRpq8RliqHY5uOVWiHfZoNtdOXIsNvzVvq5FqlHOWt71ofxXTtlg== dependencies: "@libp2p/interface-peer-id" "^2.0.0" "@libp2p/interface-peer-info" "^1.0.0" "@libp2p/interfaces" "^3.0.0" -"@libp2p/interface-peer-store@^1.0.0", "@libp2p/interface-peer-store@^1.2.1", "@libp2p/interface-peer-store@^1.2.2": - version "1.2.8" - resolved "https://registry.yarnpkg.com/@libp2p/interface-peer-store/-/interface-peer-store-1.2.8.tgz#d36ca696cf4ac377dbdd13b132a378f161e64ad3" - integrity sha512-FM9VLmpg9CUBKZ2RW+J7RrQfQVMksLiC8oqENqHgb/VkPJY3kafbn7HIi0NcK6H/H5VcwBIhL15SUJk66O1K6g== +"@libp2p/interface-peer-store@^2.0.0", "@libp2p/interface-peer-store@^2.0.3", "@libp2p/interface-peer-store@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@libp2p/interface-peer-store/-/interface-peer-store-2.0.4.tgz#5e9961b37094341216301285edf6fd73f3e796aa" + integrity sha512-jNvBK3O1JPJqSiDN2vkb+PV8bTPnYdP54nxsLtut1BWukNm610lwzwleV7CetFI4bJCn6g+BgBvvq8fdADy0tA== dependencies: "@libp2p/interface-peer-id" "^2.0.0" - "@libp2p/interface-peer-info" "^1.0.0" - "@libp2p/interface-record" "^2.0.0" - "@libp2p/interfaces" "^3.0.0" - "@multiformats/multiaddr" "^11.0.0" + "@multiformats/multiaddr" "^12.0.0" -"@libp2p/interface-pubsub@^3.0.0": - version "3.0.6" - resolved "https://registry.yarnpkg.com/@libp2p/interface-pubsub/-/interface-pubsub-3.0.6.tgz#416f52d44ebc7e62e6b5caf086aff3e429e4a950" - integrity sha512-c1aVHAhxmEh9IpLBgJyCsMscVDl7YUeP1Iq6ILEQoWiPJhNpQqdfmqyk7ZfrzuBU19VFe1EqH0bLuLDbtfysTQ== +"@libp2p/interface-pubsub@^4.0.0", "@libp2p/interface-pubsub@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@libp2p/interface-pubsub/-/interface-pubsub-4.0.1.tgz#27f85b43ced13cf3382629a38f309f7fc7b45bec" + integrity sha512-PIc5V/J98Yr1ZTHh8lQshP7GdVUh+pKNIqj6wGaDmXs8oQLB40qKCjcpHQNlAnv2e1Bh9mEH2GXv5sGZOA651A== dependencies: - "@libp2p/interface-connection" "^3.0.0" + "@libp2p/interface-connection" "^5.0.0" "@libp2p/interface-peer-id" "^2.0.0" "@libp2p/interfaces" "^3.0.0" - it-pushable "^3.0.0" - uint8arraylist "^2.1.2" + it-pushable "^3.1.3" + uint8arraylist "^2.4.3" -"@libp2p/interface-record@^2.0.0", "@libp2p/interface-record@^2.0.1": - version "2.0.6" - resolved "https://registry.yarnpkg.com/@libp2p/interface-record/-/interface-record-2.0.6.tgz#44597e144bc3e9960cc64f8c5fcd9822ea3e283f" - integrity sha512-4EtDkY3sbYapWM8++gVHlv31HZXoLmj9I7CRXUKXzFkVE0GLK/A8jYWl7K0lmf2juPjeYm2eHITeA9/wAtIS3w== +"@libp2p/interface-record@^2.0.1", "@libp2p/interface-record@^2.0.6": + version "2.0.7" + resolved "https://registry.yarnpkg.com/@libp2p/interface-record/-/interface-record-2.0.7.tgz#d083776e465cfa66d10e1d3c8e015677a9fc7635" + integrity sha512-AFPytZWI+p8FJWP0xuK5zbSjalLAOIMzEed2lBKdRWvdGBQUHt9ENLTkfkI9G7p/Pp3hlhVzzBXdIErKd+0GxQ== dependencies: "@libp2p/interface-peer-id" "^2.0.0" - uint8arraylist "^2.1.2" + uint8arraylist "^2.4.3" -"@libp2p/interface-registrar@^2.0.0", "@libp2p/interface-registrar@^2.0.3", "@libp2p/interface-registrar@^2.0.8": - version "2.0.8" - resolved "https://registry.yarnpkg.com/@libp2p/interface-registrar/-/interface-registrar-2.0.8.tgz#81038a9a814a20dba1d75ba66a45a33b98bd0a98" - integrity sha512-WbnXB09QF41zZzNgDUAZrRMilqgB7wBMTsSvql8xdDcws+jbaX4wE0iEpRXg1hyd0pz4mooIcMRaH1NiEQ5D8w== +"@libp2p/interface-registrar@^2.0.0", "@libp2p/interface-registrar@^2.0.11", "@libp2p/interface-registrar@^2.0.12", "@libp2p/interface-registrar@^2.0.3": + version "2.0.12" + resolved "https://registry.yarnpkg.com/@libp2p/interface-registrar/-/interface-registrar-2.0.12.tgz#a74b59df7b6c345d8bb45d310469b2d5f923e9bf" + integrity sha512-EyCi2bycC2rn3oPB4Swr7EqBsvcaWd6RcqR6zsImNIG9BKc4/R1gl6iaF861JaELYgYmzBMS31x1rQpVz5UekQ== dependencies: - "@libp2p/interface-connection" "^3.0.0" + "@libp2p/interface-connection" "^5.0.0" "@libp2p/interface-peer-id" "^2.0.0" -"@libp2p/interface-stream-muxer@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@libp2p/interface-stream-muxer/-/interface-stream-muxer-3.0.0.tgz#cd884a8390917170873eb0a8894fae51bf16ef9f" - integrity sha512-qv7Z4KJC2SLu/GhwDzT71VBHhtu2fpSL/DGh0iFmkxicQsMmdpiqmXv9EGKw3+jdQL57uKIUm98OpOi2Hge0kg== - dependencies: - "@libp2p/interface-connection" "^3.0.0" - "@libp2p/interfaces" "^3.0.0" - it-stream-types "^1.0.4" - -"@libp2p/interface-transport@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@libp2p/interface-transport/-/interface-transport-2.0.0.tgz#f2c7c474ea24cfb16dbc830c8d28eaa646c0041c" - integrity sha512-aK33gpkEzfEtoSPlS7z624Tubf49CD2x4DUbXOmQCXDP/rhh3gAQq5XU5dcincM3QXlx6RRSO1PWRqM8EnLE0Q== +"@libp2p/interface-stream-muxer@^4.0.0", "@libp2p/interface-stream-muxer@^4.1.2": + version "4.1.2" + resolved "https://registry.yarnpkg.com/@libp2p/interface-stream-muxer/-/interface-stream-muxer-4.1.2.tgz#f0a5edb906ec784d991b9421a024f0f21ebdaab4" + integrity sha512-dQJcn67UaAa8YQFRJDhbo4uT453z/2lCzD/ZwTk1YOqJxATXbXgVcB8dXDQFEUiUX3ZjVQ1IBu+NlQd+IZ++zw== dependencies: - "@libp2p/interface-connection" "^3.0.0" - "@libp2p/interface-stream-muxer" "^3.0.0" + "@libp2p/interface-connection" "^5.0.0" "@libp2p/interfaces" "^3.0.0" - "@multiformats/multiaddr" "^11.0.0" - it-stream-types "^1.0.4" + "@libp2p/logger" "^2.0.7" + abortable-iterator "^5.0.1" + any-signal "^4.1.1" + it-pushable "^3.1.3" + it-stream-types "^2.0.1" + uint8arraylist "^2.4.3" -"@libp2p/interface-transport@^2.1.0": - version "2.1.1" - resolved "https://registry.yarnpkg.com/@libp2p/interface-transport/-/interface-transport-2.1.1.tgz#e463f30b272494c177d3a0bd494545616fd7b624" - integrity sha512-xDM/s8iPN/XfNqD9qNelibRMPKkhOLinXwQeNtoTZjarq+Cg6rtO6/5WBG/49hyI3+r+5jd2eykjPGQbb86NFQ== +"@libp2p/interface-transport@^4.0.0": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@libp2p/interface-transport/-/interface-transport-4.0.3.tgz#8cc63bb4863ece507cbc54bff167fc7588fd3a85" + integrity sha512-jXFQ3blhFMEyQbFw/U8Glo3F/fUO5LEaX5HIdeqNpCliK+XnwTfpkcaG+WsJrcApWK4FFyUHc+GGqiWR0hAFFg== dependencies: - "@libp2p/interface-connection" "^3.0.0" - "@libp2p/interface-stream-muxer" "^3.0.0" + "@libp2p/interface-connection" "^5.0.0" + "@libp2p/interface-stream-muxer" "^4.0.0" "@libp2p/interfaces" "^3.0.0" - "@multiformats/multiaddr" "^11.0.0" - it-stream-types "^1.0.4" + "@multiformats/multiaddr" "^12.0.0" + it-stream-types "^2.0.1" "@libp2p/interfaces@^3.0.0", "@libp2p/interfaces@^3.0.2", "@libp2p/interfaces@^3.0.3", "@libp2p/interfaces@^3.2.0", "@libp2p/interfaces@^3.3.1": version "3.3.1" resolved "https://registry.yarnpkg.com/@libp2p/interfaces/-/interfaces-3.3.1.tgz#519c77c030b10d776250bbebf65990af53ccb2ee" integrity sha512-3N+goQt74SmaVOjwpwMPKLNgh1uDQGw8GD12c40Kc86WOq0qvpm3NfACW+H8Su2X6KmWjCSMzk9JWs9+8FtUfg== -"@libp2p/logger@^2.0.0", "@libp2p/logger@^2.0.1", "@libp2p/logger@^2.0.2", "@libp2p/logger@^2.0.5": - version "2.0.5" - resolved "https://registry.yarnpkg.com/@libp2p/logger/-/logger-2.0.5.tgz#cf0ee695ba21471fd085a7fda3e534e03946ad20" - integrity sha512-WEhxsc7+gsfuTcljI4vSgW/H2f18aBaC+JiO01FcX841Wxe9szjzHdBLDh9eqygUlzoK0LEeIBfctN7ibzus5A== +"@libp2p/keychain@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@libp2p/keychain/-/keychain-2.0.0.tgz#8776233e9cccabc197963a8e54d10fb014b35120" + integrity sha512-BJMqZCR6Bt3snxOeszKr/3+Y35pb3hZkuiaVP7vXfC5ID9RuFRGqAHdGzz+FVqow1XZSuUTNrL/NydF1TvJHRw== dependencies: - "@libp2p/interface-peer-id" "^2.0.0" - debug "^4.3.3" - interface-datastore "^7.0.0" - multiformats "^11.0.0" + "@libp2p/crypto" "^1.0.11" + "@libp2p/interface-keychain" "^2.0.3" + "@libp2p/interface-peer-id" "^2.0.1" + "@libp2p/interfaces" "^3.3.1" + "@libp2p/logger" "^2.0.5" + "@libp2p/peer-id" "^2.0.1" + interface-datastore "^8.0.0" + merge-options "^3.0.4" + sanitize-filename "^1.6.3" + uint8arrays "^4.0.3" -"@libp2p/mdns@^6.0.0": - version "6.0.0" - resolved "https://registry.yarnpkg.com/@libp2p/mdns/-/mdns-6.0.0.tgz#dc68881a58c0ce5e4cc3490c0c9d74d1e1b88936" - integrity sha512-k5Gi0IrPi3roPHF71xlq9x69TxqjMNZ+JiM7hFV0kjRYmaLqYQ+dOTLJLUd5ZfnrxIe8KkapFw3zwKne4Dw4rA== +"@libp2p/logger@^2.0.0", "@libp2p/logger@^2.0.1", "@libp2p/logger@^2.0.2", "@libp2p/logger@^2.0.5", "@libp2p/logger@^2.0.7", "@libp2p/logger@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@libp2p/logger/-/logger-2.1.1.tgz#e12e6c320ea64252af954bcec996895098d1cd36" + integrity sha512-2UbzDPctg3cPupF6jrv6abQnAUTrbLybNOj0rmmrdGm1cN2HJ1o/hBu0sXuq4KF9P1h/eVRn1HIRbVIEKnEJrA== dependencies: - "@libp2p/interface-peer-discovery" "^1.0.1" - "@libp2p/interface-peer-id" "^2.0.0" - "@libp2p/interface-peer-info" "^1.0.3" - "@libp2p/interfaces" "^3.0.3" - "@libp2p/logger" "^2.0.1" - "@libp2p/peer-id" "^2.0.0" - "@multiformats/multiaddr" "^11.0.0" + "@libp2p/interface-peer-id" "^2.0.2" + "@multiformats/multiaddr" "^12.1.3" + debug "^4.3.4" + interface-datastore "^8.2.0" + multiformats "^11.0.2" + +"@libp2p/mdns@^8.0.0": + version "8.0.0" + resolved "https://registry.yarnpkg.com/@libp2p/mdns/-/mdns-8.0.0.tgz#250e498bb0b5d3a4b253460ef310598033b4b2c0" + integrity sha512-/q2qDWGzZpv2/LmvlwsImoEwjOhmaO9H7HDFloEs2D1+rT0dRFuQpXHAm7/sCLwx9PtmSUZp/sNj0ppnGGwK5A== + dependencies: + "@libp2p/interface-peer-discovery" "^2.0.0" + "@libp2p/interface-peer-info" "^1.0.8" + "@libp2p/interfaces" "^3.3.1" + "@libp2p/logger" "^2.0.5" + "@libp2p/peer-id" "^2.0.1" + "@multiformats/multiaddr" "^12.0.0" "@types/multicast-dns" "^7.2.1" - multicast-dns "^7.2.0" - multiformats "^11.0.0" + dns-packet "^5.4.0" + multicast-dns "^7.2.5" -"@libp2p/mplex@^7.1.3": - version "7.1.3" - resolved "https://registry.yarnpkg.com/@libp2p/mplex/-/mplex-7.1.3.tgz#5f9e93844e0adac848a00e9d870d33283f6564ef" - integrity sha512-CBjN6otOYyJzogS0rgMnE7NwIC0c6OP6qLc/LybOcI16PJHn1gP25QWrWfNOl+K5M92ybwykMnB2cPi3blLH6Q== +"@libp2p/mplex@^8.0.3": + version "8.0.3" + resolved "https://registry.yarnpkg.com/@libp2p/mplex/-/mplex-8.0.3.tgz#cd191866a5bc4c2870f6b446d40f0d0396639352" + integrity sha512-qMaMHmjYxkInQKRgBx1bsJB9T4FPqbvkwU9oItARl134Xila0ZqGaiRdy7m4aBVY0jmd0Jfq0F1ysy6KUCXxFA== dependencies: - "@libp2p/interface-connection" "^3.0.1" - "@libp2p/interface-stream-muxer" "^3.0.0" + "@libp2p/interface-connection" "^5.0.0" + "@libp2p/interface-stream-muxer" "^4.1.2" "@libp2p/interfaces" "^3.2.0" "@libp2p/logger" "^2.0.0" - abortable-iterator "^4.0.2" - any-signal "^3.0.0" + abortable-iterator "^5.0.0" + any-signal "^4.0.1" benchmark "^2.1.4" - it-batched-bytes "^1.0.0" + it-batched-bytes "^2.0.2" it-pushable "^3.1.0" - it-stream-types "^1.0.4" + it-stream-types "^2.0.1" rate-limiter-flexible "^2.3.9" uint8arraylist "^2.1.1" uint8arrays "^4.0.2" varint "^6.0.0" -"@libp2p/multistream-select@^3.0.0": - version "3.1.2" - resolved "https://registry.yarnpkg.com/@libp2p/multistream-select/-/multistream-select-3.1.2.tgz#2302ac57daa443ceced8481a83c58e39ab601b3f" - integrity sha512-NfF0fwQM4sqiLuNGBVc9z2mfz3OigOfyLJ5zekRBGYHkbKWrBRFS3FligUPr9roCOzH6ojjDkKVd5aK9/llfJQ== +"@libp2p/multistream-select@^3.1.8": + version "3.1.9" + resolved "https://registry.yarnpkg.com/@libp2p/multistream-select/-/multistream-select-3.1.9.tgz#60b12503bab879a2ebb97d69f4670a10e67c35c8" + integrity sha512-iSNqr8jXvOrkNTyA43h/ARs4wd0Rd55/D6oFRndLcV4yQSUMmfjl7dUcbC5MAw+5/sgskfDx9TMawSwNq47Qwg== dependencies: - "@libp2p/interfaces" "^3.0.2" + "@libp2p/interfaces" "^3.2.0" "@libp2p/logger" "^2.0.0" - abortable-iterator "^4.0.2" - err-code "^3.0.1" - it-first "^2.0.0" - it-handshake "^4.1.2" - it-length-prefixed "^8.0.3" - it-merge "^2.0.0" - it-pipe "^2.0.4" + abortable-iterator "^5.0.0" + it-first "^3.0.1" + it-handshake "^4.1.3" + it-length-prefixed "^9.0.0" + it-merge "^3.0.0" + it-pipe "^3.0.0" it-pushable "^3.1.0" it-reader "^6.0.1" - it-stream-types "^1.0.4" - p-defer "^4.0.0" + it-stream-types "^2.0.1" uint8arraylist "^2.3.1" uint8arrays "^4.0.2" -"@libp2p/peer-collections@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@libp2p/peer-collections/-/peer-collections-3.0.0.tgz#dd1eeb5f562d857f23dbe95b13d595b13c273d04" - integrity sha512-rVhfDmkVzfBVR4scAfaKb05htZENx01PYt2USi1EnODyoo2c2U2W5tfOfyaKI/4D+ayQDOjT27G0ZCyAgwkYGw== - dependencies: - "@libp2p/interface-peer-id" "^2.0.0" - "@libp2p/peer-id" "^2.0.0" - -"@libp2p/peer-id-factory@^2.0.0", "@libp2p/peer-id-factory@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@libp2p/peer-id-factory/-/peer-id-factory-2.0.1.tgz#36d92e0ae55f039812224c7dcf42e16aa3bab039" - integrity sha512-CRJmqwNQhDC51sQ9lf6EqEY8HuywwymMVffL2kIYI5ts5k+6gvIXzoSxLf3V3o+OxcroXG4KG0uGxxAi5DUXSA== +"@libp2p/peer-collections@^3.0.0", "@libp2p/peer-collections@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@libp2p/peer-collections/-/peer-collections-3.0.1.tgz#77080198e6222fcb6d8633aa5a3feeb9afcb3196" + integrity sha512-tJvCjFSKX76VacThVnN0XC4jnUeufYD2u9TxWJllSYnmmos/Lwhl4kdtEyZkKNlJKam+cBoUmODXzasdoPZgVg== dependencies: - "@libp2p/crypto" "^1.0.0" - "@libp2p/interface-keys" "^1.0.2" "@libp2p/interface-peer-id" "^2.0.0" "@libp2p/peer-id" "^2.0.0" - multiformats "^11.0.0" - protons-runtime "^4.0.1" - uint8arraylist "^2.0.0" - uint8arrays "^4.0.2" -"@libp2p/peer-id-factory@^2.0.3": +"@libp2p/peer-id-factory@^2.0.0", "@libp2p/peer-id-factory@^2.0.3": version "2.0.3" resolved "https://registry.yarnpkg.com/@libp2p/peer-id-factory/-/peer-id-factory-2.0.3.tgz#d841989494c4900e01f6e3929ef06b8cc4e56f8f" integrity sha512-9pwVbfghiKuiC76Pue/+tI4PD7gnw1jGVcxYD+nhcRs8ABE7NLaB7nCm99cCtvmMNRnl2JqaGgZJXt8mnvAEuQ== @@ -2150,122 +2132,107 @@ uint8arraylist "^2.0.0" uint8arrays "^4.0.2" -"@libp2p/peer-id@^2.0.0", "@libp2p/peer-id@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@libp2p/peer-id/-/peer-id-2.0.1.tgz#1cfa5a51a3adcf91489d88c5b75d3cf6f03e2ab4" - integrity sha512-uGIR4rS+j+IzzIu0kih4MonZEfRmjGNfXaSPMIFOeMxZItZT6TIpxoVNYxHl4YtneSFKzlLnf9yx9EhRcyfy8Q== +"@libp2p/peer-id@^2.0.0", "@libp2p/peer-id@^2.0.1", "@libp2p/peer-id@^2.0.3": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@libp2p/peer-id/-/peer-id-2.0.3.tgz#7299d74eae7b2526123d941bdb2d08462704c79a" + integrity sha512-eZX+5ByUAzh8DrfjCan0spZGpvF7SxEBz4tOPoBMBCuKJJLr+8EokBO/5E3ceIw04f5+lAcD3CO3bccuKomp3Q== dependencies: "@libp2p/interface-peer-id" "^2.0.0" "@libp2p/interfaces" "^3.2.0" multiformats "^11.0.0" uint8arrays "^4.0.2" -"@libp2p/peer-record@^5.0.0": - version "5.0.0" - resolved "https://registry.yarnpkg.com/@libp2p/peer-record/-/peer-record-5.0.0.tgz#c4d472a5b7fc7e728636e114928dace3a1f12cc9" - integrity sha512-qGaqYQSRqI/vol1NEMR9Z3ncLjIkyIF0o/CQYXzXCDjA91i9+0iMjXGgIgBLn3bfA1b9pHuz4HvwjgYUKMYOkQ== +"@libp2p/peer-record@^5.0.0", "@libp2p/peer-record@^5.0.3": + version "5.0.3" + resolved "https://registry.yarnpkg.com/@libp2p/peer-record/-/peer-record-5.0.3.tgz#eceb3ed6419e0cade035542540115fb5c14f647b" + integrity sha512-KnQR/NteL0xGKXd9rZo/W3ZT9kajmNy98/BOOlnMktkAL7jCfHy2z/laDU+rSttTy1TYZ15zPzXtnm3813ECmg== dependencies: "@libp2p/crypto" "^1.0.11" "@libp2p/interface-peer-id" "^2.0.0" "@libp2p/interface-record" "^2.0.1" - "@libp2p/logger" "^2.0.5" + "@libp2p/interfaces" "^3.2.0" "@libp2p/peer-id" "^2.0.0" "@libp2p/utils" "^3.0.0" - "@multiformats/multiaddr" "^11.0.0" - err-code "^3.0.1" - interface-datastore "^7.0.0" - it-all "^2.0.0" - it-filter "^2.0.0" - it-foreach "^1.0.0" - it-map "^2.0.0" - it-pipe "^2.0.3" - multiformats "^11.0.0" - protons-runtime "^4.0.1" + "@multiformats/multiaddr" "^12.0.0" + protons-runtime "^5.0.0" uint8-varint "^1.0.2" uint8arraylist "^2.1.0" uint8arrays "^4.0.2" - varint "^6.0.0" -"@libp2p/peer-store@^6.0.0": - version "6.0.0" - resolved "https://registry.yarnpkg.com/@libp2p/peer-store/-/peer-store-6.0.0.tgz#28461ffc018f491d9b7313e284ba582fe75a116c" - integrity sha512-7GSqRYkJR3E0Vo96XH84X6KNPdwOE1t6jb7jegYzvzKDZMFaceJUZg9om3+ZHCUbethnYuqsY7j0c7OHCB40nA== +"@libp2p/peer-store@^8.2.0": + version "8.2.0" + resolved "https://registry.yarnpkg.com/@libp2p/peer-store/-/peer-store-8.2.0.tgz#b2faa4fdff91669a983e30b81128ee33195c6550" + integrity sha512-6QVT16ThxVmGHxTRmT5vKF8d2zVkG+ioxLO51Z+NrFMi/UgqTffx0qozWOcJ3CFqxPOS7MLR+wFecej2eGu2/w== dependencies: + "@libp2p/interface-libp2p" "^3.1.0" "@libp2p/interface-peer-id" "^2.0.0" - "@libp2p/interface-peer-info" "^1.0.3" - "@libp2p/interface-peer-store" "^1.2.2" - "@libp2p/interface-record" "^2.0.1" - "@libp2p/interfaces" "^3.0.3" - "@libp2p/logger" "^2.0.0" + "@libp2p/interface-peer-store" "^2.0.4" + "@libp2p/interfaces" "^3.2.0" + "@libp2p/logger" "^2.0.7" + "@libp2p/peer-collections" "^3.0.1" "@libp2p/peer-id" "^2.0.0" - "@libp2p/peer-record" "^5.0.0" - "@multiformats/multiaddr" "^11.0.0" - err-code "^3.0.1" - interface-datastore "^7.0.0" - it-all "^2.0.0" - it-filter "^2.0.0" - it-foreach "^1.0.0" - it-map "^2.0.0" - it-pipe "^2.0.3" - mortice "^3.0.0" + "@libp2p/peer-id-factory" "^2.0.0" + "@libp2p/peer-record" "^5.0.3" + "@multiformats/multiaddr" "^12.0.0" + interface-datastore "^8.0.0" + it-all "^3.0.2" + mortice "^3.0.1" multiformats "^11.0.0" - protons-runtime "^4.0.1" + protons-runtime "^5.0.0" uint8arraylist "^2.1.1" uint8arrays "^4.0.2" -"@libp2p/prometheus-metrics@^1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@libp2p/prometheus-metrics/-/prometheus-metrics-1.1.3.tgz#a884e282598c7d693674658a21c05d65fa6c9e6b" - integrity sha512-4rpn+ND/w2y5oWdP15KADLpD+SX6hdnEN0eZf+L18p8MaMr5qjpQBsCTe51VitKfVy5kIujNlquSVLhPozWFAA== +"@libp2p/prometheus-metrics@^1.1.4": + version "1.1.4" + resolved "https://registry.yarnpkg.com/@libp2p/prometheus-metrics/-/prometheus-metrics-1.1.4.tgz#ad490f0bfb63f2db7cadc65cf7f8a999c7396af2" + integrity sha512-PeszZJQKliGGLLjni7QP3OiOKY0qNbAIo1ySBSFjkg4HL9UYK/12kYQRR7tbsjOhnjDffNzOs4oYli9s0j9/7g== dependencies: - "@libp2p/interface-connection" "^3.0.2" + "@libp2p/interface-connection" "^5.0.2" "@libp2p/interface-metrics" "^4.0.2" "@libp2p/logger" "^2.0.2" - it-foreach "^1.0.0" - it-stream-types "^1.0.4" + it-foreach "^2.0.3" + it-stream-types "^2.0.1" -"@libp2p/pubsub@^6.0.0": - version "6.0.0" - resolved "https://registry.yarnpkg.com/@libp2p/pubsub/-/pubsub-6.0.0.tgz#8072ff511e901e5c0bb4226fa14f9529315af01d" - integrity sha512-WWViQ+fEL3JWt415UznUR6wQCm+UCi65SNQWQoTRYaCM2DYVCrIRfGpmFWAyKPCr76L6UesucIkZHuyh2c3xNA== +"@libp2p/pubsub@^7.0.1": + version "7.0.1" + resolved "https://registry.yarnpkg.com/@libp2p/pubsub/-/pubsub-7.0.1.tgz#8b9aa8aff7485ab0e1c451f0e24a898f226bb934" + integrity sha512-CC/d1BwIT/K/lHaubwsGfo5smdWO9enwQROlSK0RdxXwPD8psf9y8YAN+cmxJa+Xls+Qhq2YU1f9bpSrjSkOtA== dependencies: "@libp2p/crypto" "^1.0.0" - "@libp2p/interface-connection" "^3.0.1" - "@libp2p/interface-peer-id" "^2.0.0" - "@libp2p/interface-pubsub" "^3.0.0" - "@libp2p/interface-registrar" "^2.0.0" - "@libp2p/interfaces" "^3.0.2" - "@libp2p/logger" "^2.0.0" - "@libp2p/peer-collections" "^3.0.0" - "@libp2p/peer-id" "^2.0.0" - "@libp2p/topology" "^4.0.0" - "@multiformats/multiaddr" "^11.0.0" - abortable-iterator "^4.0.2" - err-code "^3.0.1" - it-length-prefixed "^8.0.2" - it-pipe "^2.0.3" - it-pushable "^3.0.0" + "@libp2p/interface-connection" "^5.0.1" + "@libp2p/interface-peer-id" "^2.0.1" + "@libp2p/interface-pubsub" "^4.0.0" + "@libp2p/interface-registrar" "^2.0.11" + "@libp2p/interfaces" "^3.3.1" + "@libp2p/logger" "^2.0.7" + "@libp2p/peer-collections" "^3.0.1" + "@libp2p/peer-id" "^2.0.3" + "@libp2p/topology" "^4.0.1" + abortable-iterator "^5.0.1" + it-length-prefixed "^9.0.0" + it-pipe "^3.0.1" + it-pushable "^3.1.3" multiformats "^11.0.0" p-queue "^7.2.0" uint8arraylist "^2.0.0" uint8arrays "^4.0.2" -"@libp2p/tcp@6.1.0": - version "6.1.0" - resolved "https://registry.yarnpkg.com/@libp2p/tcp/-/tcp-6.1.0.tgz#f4da4eb5974cf91b25c8395994d4c68a50ebc73a" - integrity sha512-eBymh9uMoj+fi88evxY2eUGny8lAVo2LE4SqjuCuqGgMbcFF3AL30eIVMyDZCmg41NAFShNtbQ9zBMqhcGqDAA== +"@libp2p/tcp@7.0.1": + version "7.0.1" + resolved "https://registry.yarnpkg.com/@libp2p/tcp/-/tcp-7.0.1.tgz#116b311deb89e70a04ab31f70c783f2224ef3d48" + integrity sha512-But+0FiTBNjIpFeYMpq5QetSLK0MKQaRjDYsYC0AUIE61TmrWE4tpxnW57rl/hKJprVbYs/9lYxgflL9Mo33Wg== dependencies: - "@libp2p/interface-connection" "^3.0.2" + "@libp2p/interface-connection" "^5.0.0" "@libp2p/interface-metrics" "^4.0.0" - "@libp2p/interface-transport" "^2.0.0" + "@libp2p/interface-transport" "^4.0.0" "@libp2p/interfaces" "^3.2.0" "@libp2p/logger" "^2.0.0" "@libp2p/utils" "^3.0.2" - "@multiformats/mafmt" "^11.0.3" - "@multiformats/multiaddr" "^11.0.0" + "@multiformats/mafmt" "^12.0.0" + "@multiformats/multiaddr" "^12.0.0" stream-to-it "^0.2.2" -"@libp2p/topology@^4.0.0": +"@libp2p/topology@^4.0.0", "@libp2p/topology@^4.0.1": version "4.0.1" resolved "https://registry.yarnpkg.com/@libp2p/topology/-/topology-4.0.1.tgz#8efab229ed32d30cfa6c4a371e8022011c0ff6f9" integrity sha512-wcToZU3o55nTPuN+yEpAublGzomGfxEAu8snaGeZS0f6ObzaQXqPgZvD5qpiQ8yOOVjR+IiNEjZJiuqNShHnaA== @@ -2282,30 +2249,23 @@ dependencies: "@libp2p/interface-metrics" "^4.0.0" -"@libp2p/utils@^3.0.0", "@libp2p/utils@^3.0.2": - version "3.0.2" - resolved "https://registry.yarnpkg.com/@libp2p/utils/-/utils-3.0.2.tgz#a65b5e5de607875f26214fc00610ac6d31451d18" - integrity sha512-/+mwCEd1o1sko3fYkVfy9pDT3Ks+KszR4Y3fb3M3/UCETDituvqZKHHM4wyTJsFlrFrohbtYlNvWhJ7Pej3X5g== +"@libp2p/utils@^3.0.0", "@libp2p/utils@^3.0.10", "@libp2p/utils@^3.0.2": + version "3.0.11" + resolved "https://registry.yarnpkg.com/@libp2p/utils/-/utils-3.0.11.tgz#d1611c3d7836eb32e5fc8bcc19c620e77471f44f" + integrity sha512-d8ZQnu2o78TG7Oy4G6qFy5v/kNBtfgQjy1RpiQAEAB6AOSi1Oq8nLebrgCqSHfrtOIcj6a+G6ImYBaRE4b03CA== dependencies: "@achingbrain/ip-address" "^8.1.0" - "@libp2p/interface-connection" "^3.0.2" - "@libp2p/interface-peer-store" "^1.2.1" + "@libp2p/interface-connection" "^5.0.1" + "@libp2p/interface-peer-store" "^2.0.0" + "@libp2p/interfaces" "^3.2.0" "@libp2p/logger" "^2.0.0" - "@multiformats/multiaddr" "^11.0.0" - abortable-iterator "^4.0.2" - err-code "^3.0.1" + "@multiformats/multiaddr" "^12.0.0" + abortable-iterator "^5.0.0" is-loopback-addr "^2.0.1" - it-stream-types "^1.0.4" - private-ip "^2.1.1" + it-stream-types "^2.0.1" + private-ip "^3.0.0" uint8arraylist "^2.3.2" -"@multiformats/mafmt@^11.0.2", "@multiformats/mafmt@^11.0.3": - version "11.0.3" - resolved "https://registry.yarnpkg.com/@multiformats/mafmt/-/mafmt-11.0.3.tgz#278bcf23ca7c954a9a04500527c011a6ce14f0cb" - integrity sha512-DvCQeZJgaC4kE3BLqMuW3gQkNAW14Z7I+yMt30Ze+wkfHkWSp+bICcHGihhtgfzYCumHA/vHlJ9n54mrCcmnvQ== - dependencies: - "@multiformats/multiaddr" "^11.0.0" - "@multiformats/mafmt@^12.0.0": version "12.1.0" resolved "https://registry.yarnpkg.com/@multiformats/mafmt/-/mafmt-12.1.0.tgz#9984f1e5314631a0472ccb91ea1ce12ea1f5059b" @@ -2313,22 +2273,23 @@ dependencies: "@multiformats/multiaddr" "^12.0.0" -"@multiformats/multiaddr@^11.0.0": - version "11.0.7" - resolved "https://registry.yarnpkg.com/@multiformats/multiaddr/-/multiaddr-11.0.7.tgz#1151e474e4a097657e4f18fd01a64a273d178a46" - integrity sha512-rCqYS3Qz/dm4H/1Lvda11OBZf1tH8rst69GWK9jDy8AY+3n+NBBdErA/SRtdcRx6hPtQ8lAB5UhHlzIVbViv1Q== +"@multiformats/multiaddr@^12.0.0": + version "12.1.1" + resolved "https://registry.yarnpkg.com/@multiformats/multiaddr/-/multiaddr-12.1.1.tgz#40f132438bc18069b3f51ca1d50313ef8456b0ea" + integrity sha512-j4mTCNSRhsoiAJ+Bp+Kj9Fv1Ij+gplfHEc9Tk53/UpZDN+oe+KN9jmIkzOEKiC3gZXpt/R7+7gw1aOoCS9feDA== dependencies: "@chainsafe/is-ip" "^2.0.1" + "@chainsafe/netmask" "^2.0.0" + "@libp2p/interfaces" "^3.3.1" dns-over-http-resolver "^2.1.0" - err-code "^3.0.1" - multiformats "^10.0.0" + multiformats "^11.0.0" uint8arrays "^4.0.2" varint "^6.0.0" -"@multiformats/multiaddr@^12.0.0": - version "12.1.1" - resolved "https://registry.yarnpkg.com/@multiformats/multiaddr/-/multiaddr-12.1.1.tgz#40f132438bc18069b3f51ca1d50313ef8456b0ea" - integrity sha512-j4mTCNSRhsoiAJ+Bp+Kj9Fv1Ij+gplfHEc9Tk53/UpZDN+oe+KN9jmIkzOEKiC3gZXpt/R7+7gw1aOoCS9feDA== +"@multiformats/multiaddr@^12.1.3": + version "12.1.3" + resolved "https://registry.yarnpkg.com/@multiformats/multiaddr/-/multiaddr-12.1.3.tgz#aff5aa61ec19c5320f0b756e88c3bbaac8d1c7af" + integrity sha512-rNcS3njkkSwuGF4x58L47jGH5kBXBfJPNsWnrt0gujhNYn6ReDt1je7vEU5/ddrVj0TStgxw+Hm+TkYDK0b60w== dependencies: "@chainsafe/is-ip" "^2.0.1" "@chainsafe/netmask" "^2.0.0" @@ -3268,29 +3229,6 @@ resolved "https://registry.npmjs.org/@stablelib/constant-time/-/constant-time-1.0.1.tgz" integrity sha512-tNOs3uD0vSJcK6z1fvef4Y+buN7DXhzHDPqRLSXUel1UfqMB1PWNsnnAezrKfEwTLpN0cGH2p9NNjs6IqeD0eg== -"@stablelib/hash@^1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@stablelib/hash/-/hash-1.0.1.tgz" - integrity sha512-eTPJc/stDkdtOcrNMZ6mcMK1e6yBbqRBaNW55XA1jU8w/7QdnCF0CmMmOD1m7VSkBR44PWrMHU2l6r8YEQHMgg== - -"@stablelib/hkdf@^1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@stablelib/hkdf/-/hkdf-1.0.1.tgz" - integrity sha512-SBEHYE16ZXlHuaW5RcGk533YlBj4grMeg5TooN80W3NpcHRtLZLLXvKyX0qcRFxf+BGDobJLnwkvgEwHIDBR6g== - dependencies: - "@stablelib/hash" "^1.0.1" - "@stablelib/hmac" "^1.0.1" - "@stablelib/wipe" "^1.0.1" - -"@stablelib/hmac@^1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@stablelib/hmac/-/hmac-1.0.1.tgz" - integrity sha512-V2APD9NSnhVpV/QMYgCVMIYKiYG6LSqw1S65wxVoirhU/51ACio6D4yDVSwMzuTJXWZoVHbDdINioBwKy5kVmA== - dependencies: - "@stablelib/constant-time" "^1.0.1" - "@stablelib/hash" "^1.0.1" - "@stablelib/wipe" "^1.0.1" - "@stablelib/int@^1.0.1": version "1.0.1" resolved "https://registry.npmjs.org/@stablelib/int/-/int-1.0.1.tgz" @@ -3319,15 +3257,6 @@ "@stablelib/binary" "^1.0.1" "@stablelib/wipe" "^1.0.1" -"@stablelib/sha256@^1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@stablelib/sha256/-/sha256-1.0.1.tgz" - integrity sha512-GIIH3e6KH+91FqGV42Kcj71Uefd/QEe7Dy42sBTeqppXV95ggCcxLTk39bEr+lZfJmp+ghsR07J++ORkRELsBQ== - dependencies: - "@stablelib/binary" "^1.0.1" - "@stablelib/hash" "^1.0.1" - "@stablelib/wipe" "^1.0.1" - "@stablelib/wipe@^1.0.1": version "1.0.1" resolved "https://registry.npmjs.org/@stablelib/wipe/-/wipe-1.0.1.tgz" @@ -4201,13 +4130,13 @@ abort-controller@^3.0.0: dependencies: event-target-shim "^5.0.0" -abortable-iterator@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/abortable-iterator/-/abortable-iterator-4.0.2.tgz#aea6a4a6a696badcbad1c9fff5a9ca85f0f286a4" - integrity sha512-SJGELER5yXr9v3kiL6mT5RZ1qlyJ9hV4nm34+vfsdIM1lp3zENQvpsqKgykpFLgRMUn3lzlizLTpiOASW05/+g== +abortable-iterator@^5.0.0, abortable-iterator@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/abortable-iterator/-/abortable-iterator-5.0.1.tgz#5d93eba6fa8287a973a9ea090c64ca08b3777780" + integrity sha512-hlZ5Z8UwqrKsJcelVPEqDduZowJPBQJ9ZhBC2FXpja3lXy8X6MoI5uMzIgmrA8+3jcVnp8TF/tx+IBBqYJNUrg== dependencies: get-iterator "^2.0.0" - it-stream-types "^1.0.3" + it-stream-types "^2.0.1" abortcontroller-polyfill@^1.7.3: version "1.7.5" @@ -4407,11 +4336,6 @@ ansi-escapes@^6.0.0: dependencies: type-fest "^3.0.0" -ansi-regex@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" - integrity sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA== - ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz" @@ -4451,10 +4375,10 @@ any-signal@3.0.1: resolved "https://registry.yarnpkg.com/any-signal/-/any-signal-3.0.1.tgz#49cae34368187a3472e31de28fb5cb1430caa9a6" integrity sha512-xgZgJtKEa9YmDqXodIgl7Fl1C8yNXr8w6gXjqK3LW4GcEiYT+6AQfJSE/8SPsEpLLmcvbv8YU+qet94UewHxqg== -any-signal@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/any-signal/-/any-signal-3.0.0.tgz" - integrity sha512-l1H1GEkGGIXVGfCtvq8N68YI7gHajmfzRdKhmb8sGyAQpLCblirLa8eB09j4uKaiwe7vodAChocUf7AT3mYq5g== +any-signal@^4.0.1, any-signal@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/any-signal/-/any-signal-4.1.1.tgz#928416c355c66899e6b2a91cad4488f0324bae03" + integrity sha512-iADenERppdC+A2YKbOXXB2WUeABLaM6qnpZ70kZbPZ1cZMMJ7eF+3CaYm+/PhBizgkzlvssC7QuHS30oOiQYWA== anymatch@~3.1.2: version "3.1.2" @@ -4471,11 +4395,6 @@ append-transform@^2.0.0: dependencies: default-require-extensions "^3.0.0" -aproba@^1.0.3: - version "1.2.0" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" - integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== - "aproba@^1.0.3 || ^2.0.0", aproba@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz" @@ -4531,14 +4450,6 @@ are-we-there-yet@^4.0.0: delegates "^1.0.0" readable-stream "^4.1.0" -are-we-there-yet@~1.1.2: - version "1.1.7" - resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz#b15474a932adab4ff8a50d9adfa7e4e926f21146" - integrity sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g== - dependencies: - delegates "^1.0.0" - readable-stream "^2.0.6" - arg@^4.1.0: version "4.1.3" resolved "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz" @@ -5717,11 +5628,6 @@ cmd-shim@^6.0.0: resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-6.0.1.tgz#a65878080548e1dca760b3aea1e21ed05194da9d" integrity sha512-S9iI9y0nKR4hwEQsVWpyxld/6kRfGepGfzff83FcaiEBpmvlbA2nnGe7Cylgrx2f/p1P5S5wpRm9oL8z1PbS3Q== -code-point-at@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" - integrity sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA== - codecov@^3.8.3: version "3.8.3" resolved "https://registry.npmjs.org/codecov/-/codecov-3.8.3.tgz" @@ -5919,7 +5825,7 @@ console-browserify@^1.1.0: resolved "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz" integrity sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA== -console-control-strings@^1.0.0, console-control-strings@^1.1.0, console-control-strings@~1.1.0: +console-control-strings@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz" integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4= @@ -6254,36 +6160,36 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" -datastore-core@^8.0.1: - version "8.0.3" - resolved "https://registry.yarnpkg.com/datastore-core/-/datastore-core-8.0.3.tgz#d14bda75184b03d319c80d94f49ff36b27a619d4" - integrity sha512-x1cAYGXnJQRDbUYF7pUBpx4bN+UP+8MOk66A30G70pVVnIG9TbkEAiapYUrwZGFdJnpZnb3HeS5Q13rsUNxIJQ== +datastore-core@^9.0.0, datastore-core@^9.1.1: + version "9.1.1" + resolved "https://registry.yarnpkg.com/datastore-core/-/datastore-core-9.1.1.tgz#613db89a9bb2624943811dd39b831125319fab79" + integrity sha512-Way+QZdrlAjLOHm7hc3r5mEIfmdkZCtb/bAWv+Mhp9FGQKSyaf8yL5oOcmp3pv+WrqdFYB7qYx7xe/FX3+zcjw== dependencies: "@libp2p/logger" "^2.0.0" err-code "^3.0.1" - interface-datastore "^7.0.0" - it-all "^2.0.0" - it-drain "^2.0.0" - it-filter "^2.0.0" - it-map "^2.0.0" - it-merge "^2.0.0" - it-pipe "^2.0.3" + interface-store "^5.0.0" + it-all "^3.0.1" + it-drain "^3.0.1" + it-filter "^3.0.0" + it-map "^3.0.1" + it-merge "^3.0.0" + it-pipe "^3.0.0" it-pushable "^3.0.0" - it-take "^2.0.0" + it-sort "^3.0.1" + it-take "^3.0.1" uint8arrays "^4.0.2" -datastore-level@*, datastore-level@^9.0.1: - version "9.0.4" - resolved "https://registry.yarnpkg.com/datastore-level/-/datastore-level-9.0.4.tgz#1e2534fef6aedda528dcb5ead7c1f4cbcbb46d3a" - integrity sha512-HKf2tVVWywdidI+94z0B5NLx4J94wTLCT1tYXXxJ58MK/Y5rdX8WVRp9XmZaODS70uxpNC8/UrvWr0iTBZwkUA== - dependencies: - abstract-level "^1.0.3" - datastore-core "^8.0.1" - interface-datastore "^7.0.0" - it-filter "^2.0.0" - it-map "^2.0.0" - it-sort "^2.0.0" - it-take "^2.0.0" +datastore-level@*, datastore-level@^10.1.1: + version "10.1.1" + resolved "https://registry.yarnpkg.com/datastore-level/-/datastore-level-10.1.1.tgz#390dc6ca17dc691947a3e81c984b4b6064812e81" + integrity sha512-4fQPf/6fIXdcC0XZPGMiNuoOmF82Vhdz+LPTmbzR+CbbnCri6eOcFdzBPnDsAAuPOCV6Zld1EhgM2cRArw1+sQ== + dependencies: + datastore-core "^9.0.0" + interface-datastore "^8.0.0" + it-filter "^3.0.0" + it-map "^3.0.1" + it-sort "^3.0.1" + it-take "^3.0.1" level "^8.0.0" date-format@^4.0.11, date-format@^4.0.13: @@ -6317,7 +6223,7 @@ debug@4.3.4, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1, debug@~4.3. dependencies: ms "2.1.2" -debug@^3.2.6, debug@^3.2.7: +debug@^3.2.7: version "3.2.7" resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz" integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== @@ -6373,11 +6279,6 @@ deep-eql@^4.1.2: dependencies: type-detect "^4.0.0" -deep-extend@^0.6.0: - version "0.6.0" - resolved "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz" - integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== - deep-is@^0.1.3: version "0.1.3" resolved "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz" @@ -6494,11 +6395,6 @@ detect-indent@^5.0.0: resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-5.0.0.tgz#3871cc0a6a002e8c3e5b3cf7f336264675f06b9d" integrity sha512-rlpvsxUtM0PQvy9iZe640/IWwWYyBsTApREbA1pHOpmOUIl9MkP/U4z7vTtg4Oaojvqhxt7sdufnT0EzGaR31g== -detect-libc@^1.0.2: - version "1.0.3" - resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz" - integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups= - detect-node@^2.0.4: version "2.1.0" resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.1.0.tgz#c9c70775a49c3d03bc2c06d9a73be550f978f8b1" @@ -6563,13 +6459,12 @@ dns-over-http-resolver@^2.1.0, dns-over-http-resolver@^2.1.1: receptacle "^1.3.2" undici "^5.12.0" -dns-packet@^4.0.0: - version "4.2.0" - resolved "https://registry.npmjs.org/dns-packet/-/dns-packet-4.2.0.tgz" - integrity sha512-bn1AKpfkFbm0MIioOMHZ5qJzl2uypdBwI4nYNsqvhjsegBhcKJUlCrMPWLx6JEezRjxZmxhtIz/FkBEur2l8Cw== +dns-packet@^5.2.2, dns-packet@^5.4.0: + version "5.6.0" + resolved "https://registry.yarnpkg.com/dns-packet/-/dns-packet-5.6.0.tgz#2202c947845c7a63c23ece58f2f70ff6ab4c2f7d" + integrity sha512-rza3UH1LwdHh9qyPXp8lkwpjSNk/AMD3dPytUoRoqnypDUhY0xvbdmVhWOfxO68frEfV9BU8V12Ez7ZsHGZpCQ== dependencies: - ip "^1.1.5" - safe-buffer "^5.1.1" + "@leichtgewicht/ip-codec" "^2.0.1" docker-compose@^0.23.19: version "0.23.19" @@ -6691,9 +6586,9 @@ electron-to-chromium@^1.4.188: integrity sha512-nPyI7oHc8T64oSqRXrAt99gNMpk0SAgPHw/o+hkNKyb5+bcdnFtZcSO9FUJES5cVkVZvo8u4qiZ1gQILl8UXsA== electron@^21.0.1: - version "21.0.1" - resolved "https://registry.yarnpkg.com/electron/-/electron-21.0.1.tgz#753669454a86a89fd70ba9614c79f789fb177c34" - integrity sha512-jLVSLakd0fO2GPnW4xXQrI93R464jeFb2ISngqRP3wpwH96XqeANkuAYLAr9TVhfQMCIWnuPROBZ+NU7nuk0WA== + version "21.4.4" + resolved "https://registry.yarnpkg.com/electron/-/electron-21.4.4.tgz#46f24eae1ff99416312f4dfecf64b021524bb8e2" + integrity sha512-N5O7y7Gtt7mDgkJLkW49ETiT8M3myZ9tNIEvGTKhpBduX4WdgMj6c3hYeYBD6XW7SvbRkWEQaTl25RNday8Xpw== dependencies: "@electron/get" "^1.14.1" "@types/node" "^16.11.26" @@ -7617,11 +7512,6 @@ fast-diff@^1.1.2: resolved "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz" integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w== -fast-fifo@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.0.0.tgz" - integrity sha512-4VEXmjxLj7sbs8J//cn2qhRap50dGzF5n8fjay8mau+Jn4hxSeR3xPFwxMaQq/pDaq7+KQk0PAbC2+nWDkJrmQ== - fast-glob@3.2.7: version "3.2.7" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.7.tgz#fd6cb7a2d7e9aa7a7846111e85a196d6b2f766a1" @@ -8165,28 +8055,6 @@ gauge@^5.0.0: strip-ansi "^6.0.1" wide-align "^1.1.5" -gauge@~2.7.3: - version "2.7.4" - resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" - integrity sha512-14x4kjc6lkD3ltw589k0NrPD6cCNTD6CWoVUNpB85+DrtONoZn+Rug6xZU5RvSC4+TZPxA5AnBibQYAvZn41Hg== - dependencies: - aproba "^1.0.3" - console-control-strings "^1.0.0" - has-unicode "^2.0.0" - object-assign "^4.1.0" - signal-exit "^3.0.0" - string-width "^1.0.1" - strip-ansi "^3.0.1" - wide-align "^1.1.0" - -gc-stats@^1.4.0: - version "1.4.0" - resolved "https://registry.npmjs.org/gc-stats/-/gc-stats-1.4.0.tgz" - integrity sha512-4FcCj9e8j8rCjvLkqRpGZBLgTC/xr9XEf5By3x77cDucWWB3pJK6FEwXZCTCbb4z8xdaOoi4owBNrvn3ciDdxA== - dependencies: - nan "^2.13.2" - node-pre-gyp "^0.13.0" - gensync@^1.0.0-beta.1: version "1.0.0-beta.2" resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz" @@ -8666,7 +8534,7 @@ has-tostringtag@^1.0.0: dependencies: has-symbols "^1.0.2" -has-unicode@2.0.1, has-unicode@^2.0.0, has-unicode@^2.0.1: +has-unicode@2.0.1, has-unicode@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" integrity sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ== @@ -8711,11 +8579,6 @@ hasha@^5.0.0: is-stream "^2.0.0" type-fest "^0.8.0" -hashlru@^2.3.0: - version "2.3.0" - resolved "https://registry.npmjs.org/hashlru/-/hashlru-2.3.0.tgz" - integrity sha512-0cMsjjIC8I+D3M44pOQdsy0OHXGLVz6Z0beRuufhKa0KfaD2wGwAev6jILzXsd3/vpnNQJmWyZtIILqM1N+n5A== - he@1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/he/-/he-1.2.0.tgz" @@ -8878,7 +8741,7 @@ humanize-ms@^1.2.1: dependencies: ms "^2.0.0" -iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4: +iconv-lite@0.4.24, iconv-lite@^0.4.24: version "0.4.24" resolved "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz" integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== @@ -8909,7 +8772,7 @@ ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== -ignore-walk@3.0.4, ignore-walk@^3.0.1: +ignore-walk@3.0.4: version "3.0.4" resolved "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.4.tgz" integrity sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ== @@ -8994,7 +8857,7 @@ inherits@2.0.3: resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= -ini@^1.3.2, ini@^1.3.4, ini@~1.3.0: +ini@^1.3.2, ini@^1.3.4: version "1.3.8" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== @@ -9061,19 +8924,19 @@ inquirer@^9.1.5: through "^2.3.6" wrap-ansi "^8.1.0" -interface-datastore@^7.0.0: - version "7.0.4" - resolved "https://registry.yarnpkg.com/interface-datastore/-/interface-datastore-7.0.4.tgz#f09ae4e2896f57f876d5d742a59e982fb3f42891" - integrity sha512-Q8LZS/jfFFHz6XyZazLTAc078SSCoa27ZPBOfobWdpDiFO7FqPA2yskitUJIhaCgxNK8C+/lMBUTBNfVIDvLiw== +interface-datastore@^8.0.0, interface-datastore@^8.2.0: + version "8.2.0" + resolved "https://registry.yarnpkg.com/interface-datastore/-/interface-datastore-8.2.0.tgz#70076985ac17dcdb35b33c2b0f957480ce6489e1" + integrity sha512-rDMAcpCGxWMubRk2YQuSEHl11bc0xcZeBZzfLvqhoZJdByUWeo7YDJUdgyRKgD6liGXVYirtDkFU9nyn9xl2hg== dependencies: - interface-store "^3.0.0" + interface-store "^5.0.0" nanoid "^4.0.0" uint8arrays "^4.0.2" -interface-store@^3.0.0: - version "3.0.4" - resolved "https://registry.yarnpkg.com/interface-store/-/interface-store-3.0.4.tgz#670d95ef45f3b7061d154c3cbfaf39a538167ad7" - integrity sha512-OjHUuGXbH4eXSBx1TF1tTySvjLldPLzRSYYXJwrEQI+XfH5JWYZofr0gVMV4F8XTwC+4V7jomDYkvGRmDSRKqQ== +interface-store@^5.0.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/interface-store/-/interface-store-5.1.0.tgz#1735cead844fe452d62c307fafbaaa1d261e6ff3" + integrity sha512-mjUwX3XSoreoxCS3sXS3pSRsGnUjl9T06KBqt/T7AgE9Sgp4diH64ZyURJKnj2T5WmCvTbC0Dm+mwQV5hfLSBQ== internal-slot@^1.0.3: version "1.0.3" @@ -9098,11 +8961,6 @@ ip-regex@^2.1.0: resolved "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz" integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk= -ip-regex@^4.0.0, ip-regex@^4.3.0: - version "4.3.0" - resolved "https://registry.npmjs.org/ip-regex/-/ip-regex-4.3.0.tgz" - integrity sha512-B9ZWJxHHOHUhUjCPrMpLD4xEq35bUTClHM1S6CBU5ixQnkZmwipwgc96vAd7AAGM9TGHvJR+Uss+/Ak6UphK+Q== - ip-regex@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-5.0.0.tgz#cd313b2ae9c80c07bd3851e12bf4fa4dc5480632" @@ -9243,13 +9101,6 @@ is-extglob@^2.1.1: resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== -is-fullwidth-code-point@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" - integrity sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw== - dependencies: - number-is-nan "^1.0.0" - is-fullwidth-code-point@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" @@ -9294,13 +9145,6 @@ is-interactive@^2.0.0: resolved "https://registry.yarnpkg.com/is-interactive/-/is-interactive-2.0.0.tgz#40c57614593826da1100ade6059778d597f16e90" integrity sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ== -is-ip@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/is-ip/-/is-ip-3.1.0.tgz#2ae5ddfafaf05cb8008a62093cf29734f657c5d8" - integrity sha512-35vd5necO7IitFPjd/YBeqwWnyDWbuLH9ZXQdMfDA8TEo7pv5X8yfrvVO3xbJbLUlERCMvf6X0hTUamQxCYJ9Q== - dependencies: - ip-regex "^4.0.0" - is-lambda@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/is-lambda/-/is-lambda-1.0.1.tgz#3d9877899e6a53efc0160504cde15f82e6f061d5" @@ -9583,159 +9427,150 @@ it-all@^2.0.0: resolved "https://registry.yarnpkg.com/it-all/-/it-all-2.0.0.tgz#6f4e5cdb71af02793072822a90bc44de901a92c3" integrity sha512-I/yi9ogTY59lFxtfsDSlI9w9QZtC/5KJt6g7CPPBJJh2xql2ZS7Ghcp9hoqDDbc4QfwQvtx8Loy0zlKQ8H5gFg== -it-all@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/it-all/-/it-all-3.0.1.tgz#b053c383b841fdbf569ecb3849e2e89cfdee7f3f" - integrity sha512-C2xYrr8KbNek9+x5V68LkKn27ehuZ+lSCWLLQQVAWf0hzADf+QM+Xw3yEFwn8yDLNInsSonCXeM7D05h1H/43g== +it-all@^3.0.0, it-all@^3.0.1, it-all@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/it-all/-/it-all-3.0.2.tgz#620b82c702c9c6d1c4caddb6407dba4a4baa970b" + integrity sha512-ujqWETXhsDbF6C+6X6fvRw5ohlowRoy/o/h9BC8D+R3JQ13oLQ153w9gSWkWupOY7omZFQbJiAL1aJo5Gwe2yw== -it-batched-bytes@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/it-batched-bytes/-/it-batched-bytes-1.0.0.tgz#8c057d5f7442d2179427bd9afef1612db0e1ccf0" - integrity sha512-OfztV9UHQmoZ6u5F4y+YOI1Z+5JAhkv3Gexc1a0B7ikcVXc3PFSKlEnHv79u+Yp/h23o3tsF9hHAhuqgHCYT2Q== +it-batched-bytes@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/it-batched-bytes/-/it-batched-bytes-2.0.3.tgz#b05cb114c5b3dfa5d6512a18225093d1a943b358" + integrity sha512-vUhr1K6NerlrSbSKpBGW9bDd3s64GPUQePWUzoUF0zkYw2ufFpCXEYCZAtJMP45n6BJNChWDYTYwxAZvQG0b0g== dependencies: - it-stream-types "^1.0.4" p-defer "^4.0.0" uint8arraylist "^2.4.1" -it-drain@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/it-drain/-/it-drain-2.0.0.tgz#724c910720a109916bce0a991cf954e8d7b4fe21" - integrity sha512-oa/5iyBtRs9UW486vPpyDTC0ee3rqx5qlrPI7txIUJcqqtiO5yVozEB6LQrl5ysQYv+P3y/dlKEqwVqlCV0SEA== - -it-filter@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/it-filter/-/it-filter-2.0.0.tgz#bc853ffdfc7c9dcfa4511e57c4f8e104180d3e27" - integrity sha512-E68+zzoNNI7MxdH1T4lUTgwpCyEnymlH349Qg2mcvsqLmYRkaZLM4NfZZ0hUuH7/5DkWXubQSDOYH396va8mpg== +it-drain@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/it-drain/-/it-drain-3.0.2.tgz#4fb2ab30119072268c68a895fa5b9f2037942c44" + integrity sha512-0hJvS/4Ktt9wT/bktmovjjMAY8r6FCsXqpL3zjqBBNwoL21VgQfguEnwbLSGuCip9Zq1vfU43cbHkmaRZdBfOg== -it-first@^1.0.7: - version "1.0.7" - resolved "https://registry.yarnpkg.com/it-first/-/it-first-1.0.7.tgz#a4bef40da8be21667f7d23e44dae652f5ccd7ab1" - integrity sha512-nvJKZoBpZD/6Rtde6FXqwDqDZGF1sCADmr2Zoc0hZsIvnE449gRFnGctxDf09Bzc/FWnHXAdaHVIetY6lrE0/g== +it-filter@^3.0.0, it-filter@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/it-filter/-/it-filter-3.0.2.tgz#19ddf6185ea21d417e6075d5796c799fa2633b69" + integrity sha512-Hhzp5anX7tmKOBqTPasBYTPlq7l4Xk4lMBfLB5GfKZnL9WCc6pr8M9Waud4nHh3s9neb4xwDWk7KQsEapgWyJw== + dependencies: + it-peekable "^3.0.0" -it-first@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/it-first/-/it-first-2.0.0.tgz#b0bba28414caa2b27b807ac15e897d4a9526940d" - integrity sha512-fzZGzVf01exFyIZXNjkpSMFr1eW2+J1K0v018tYY26Dd4f/O3pWlBTdrOBfSQRZwtI8Pst6c7eKhYczWvFs6tA== +it-first@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/it-first/-/it-first-3.0.2.tgz#6186a40ca52c6212815177346a784c1db1034cbb" + integrity sha512-QPLAM2BOkait/o6W25HvP0XTEv+Os3Ce4wET//ADNaPv+WYAHWfQwJuMu5FB8X066hA1F7LEMnULvTpE7/4yQw== -it-foreach@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/it-foreach/-/it-foreach-1.0.0.tgz#43b3f04661ef0809a4ce03150ef1f66a3f63ed23" - integrity sha512-2j5HK1P6aMwEvgL6K5nzUwOk+81B/mjt05PxiSspFEKwJnqy1LfJYlLLS6llBoM+NdoUxf6EsBCHidFGmsXvhw== +it-foreach@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/it-foreach/-/it-foreach-2.0.3.tgz#40c96680d9875805203f61fdd1064b7190a17e5a" + integrity sha512-rpkhyHMSSe9pkmTtPcDoA5+NKhMUDqddwdXakUzNn/aOIp3vNnGBH4P4xncefxZM29iwzKBnK7AGcYVYoIG8gQ== + dependencies: + it-peekable "^3.0.0" -it-handshake@^4.1.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/it-handshake/-/it-handshake-4.1.2.tgz#9261f1869ce0162810a530e88bd40d5e7ce8e0a3" - integrity sha512-Q/EvrB4KWIX5+/wO7edBK3l79Vh28+iWPGZvZSSqwAtOJnHZIvywC+JUbiXPRJVXfICBJRqFETtIJcvrqWL2Zw== +it-handshake@^4.1.3: + version "4.1.3" + resolved "https://registry.yarnpkg.com/it-handshake/-/it-handshake-4.1.3.tgz#4e6650f8eff5cb3686c6861958645289fb3dc32a" + integrity sha512-V6Lt9A9usox9iduOX+edU1Vo94E6v9Lt9dOvg3ubFaw1qf5NCxXLi93Ao4fyCHWDYd8Y+DUhadwNtWVyn7qqLg== dependencies: it-pushable "^3.1.0" it-reader "^6.0.1" - it-stream-types "^1.0.4" + it-stream-types "^2.0.1" p-defer "^4.0.0" uint8arraylist "^2.0.0" -it-length-prefixed@^8.0.2, it-length-prefixed@^8.0.3: - version "8.0.4" - resolved "https://registry.yarnpkg.com/it-length-prefixed/-/it-length-prefixed-8.0.4.tgz#80bd356d93d77a8989a71200f8ca0860db040404" - integrity sha512-5OJ1lxH+IaqJB7lxe8IAIwt9UfSfsmjKJoAI/RO9djYoBDt1Jfy9PeVHUmOfqhqyu/4kJvWBFAJUaG1HhLQ12A== +it-length-prefixed@^9.0.0, it-length-prefixed@^9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/it-length-prefixed/-/it-length-prefixed-9.0.1.tgz#12b7f8a283251bf74102c1c92d61b33985089e7c" + integrity sha512-ZBD8ZFLERj8d1q9CeBtk0eJ4EpeI3qwnkmWtemBSm3ZI2dM8PUweNVk5haZ2vw3EIq2uYQiabV9YwNm6EASM4A== dependencies: err-code "^3.0.1" - it-stream-types "^1.0.4" + it-stream-types "^2.0.1" uint8-varint "^1.0.1" uint8arraylist "^2.0.0" uint8arrays "^4.0.2" -it-length-prefixed@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/it-length-prefixed/-/it-length-prefixed-9.0.0.tgz#df308a31b535251231c62043bf70819a1a9db28f" - integrity sha512-LCne3R3wxxLv94GTA8ywIeopdyA+2oKXiWWo7g58sQHiD7d1A6WMuWCrwP+xv4i7CmSuR3aeHo66SJUgArLOyA== +it-map@^3.0.1, it-map@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/it-map/-/it-map-3.0.3.tgz#42be39fc68dc9b0d70cfd8ac4b8311d4b5cd7f22" + integrity sha512-Yf89GJYeYUZb2NZzWkvFHm3IBXlxro74i2vGRmpf8BYau3BhlaS37ieDenJEdYzkTGJhL/EbM1jPPw/KGVVVIw== dependencies: - err-code "^3.0.1" - it-stream-types "^1.0.5" - uint8-varint "^1.0.1" - uint8arraylist "^2.0.0" - uint8arrays "^4.0.2" - -it-map@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/it-map/-/it-map-2.0.0.tgz#4fd20dfae9eeb21b3dac812774c09d490ee5b691" - integrity sha512-mLgtk/NZaN7NZ06iLrMXCA6jjhtZO0vZT5Ocsp31H+nsGI18RSPVmUbFyA1sWx7q+g92J22Sixya7T2QSSAwfA== + it-peekable "^3.0.0" -it-merge@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/it-merge/-/it-merge-2.0.0.tgz#adfcd33199995a503cb37ac73548f65d8a0742db" - integrity sha512-mH4bo/ZrMoU+Wlu7ZuYPNNh9oWZ/GvYbeXZ0zll97+Rp6H4jFu98iu6v9qqXDz//RUjdO9zGh8awzMfOElsjpA== +it-merge@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/it-merge/-/it-merge-3.0.1.tgz#20cc293593586e5afcbfed8ba88a94def5ccfcfa" + integrity sha512-I6hjU1ABO+k3xY1H6JtCSDXvUME88pxIXSgKeT4WI5rPYbQzpr98ldacVuG95WbjaJxKl6Qot6lUdxduLBQPHA== dependencies: it-pushable "^3.1.0" it-pair@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/it-pair/-/it-pair-2.0.2.tgz#1e6c7f35e8042019942b8996c0c13784d576e2d7" - integrity sha512-QGgUwGtzE4mI8yPZawL+9wq49SBmhQdjKW+ChKBm4PUwRNdkgSoyPlu280iNyS0JscBG3pvytJ8JNVPSEBQNjg== + version "2.0.6" + resolved "https://registry.yarnpkg.com/it-pair/-/it-pair-2.0.6.tgz#072defa6b96f611af34e0b0c84573107ddb9f28f" + integrity sha512-5M0t5RAcYEQYNG5BV7d7cqbdwbCAp5yLdzvkxsZmkuZsLbTdZzah6MQySYfaAQjNDCq6PUnDt0hqBZ4NwMfW6g== dependencies: - it-stream-types "^1.0.3" + it-stream-types "^2.0.1" p-defer "^4.0.0" -it-pb-stream@^3.2.0: - version "3.2.1" - resolved "https://registry.yarnpkg.com/it-pb-stream/-/it-pb-stream-3.2.1.tgz#58ad0b1268894d6eb05c17110e22326a33884a46" - integrity sha512-vKE04Zv5MUcwxPNE9bIEfYK3rd/Klj5ORGD1D8Bn5f0mbCLGfouSrqZP1Jntg2osqQg4BN5dKKS2BbfwyGUI3Q== +it-parallel@^3.0.0: + version "3.0.3" + resolved "https://registry.yarnpkg.com/it-parallel/-/it-parallel-3.0.3.tgz#b02f1d6459418c7253ecf13e367111560a616491" + integrity sha512-Q5KmdvERHCOLDcgKqrzQ+yiMCdG6H9h7ZL3Zjx/Tx9xhZy8txSKoy+EiCgWZFs0rfYvxJhk6UkOpKLzJ1zM9ZA== + dependencies: + p-defer "^4.0.0" + +it-pb-stream@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/it-pb-stream/-/it-pb-stream-4.0.1.tgz#28825106d0dcb99576d8c78d21236053b9e0f7e7" + integrity sha512-xFYnnChsx4imzxI5eBP31bJ+2+vFYS9akHQNMM8suFd+DKWOqMlxiJvcqZEkciBXCB3Wj8HF8Wyx5baSxn31gg== dependencies: err-code "^3.0.1" it-length-prefixed "^9.0.0" it-pushable "^3.1.2" - it-stream-types "^1.0.4" + it-stream-types "^2.0.1" protons-runtime "^5.0.0" uint8-varint "^1.0.6" uint8arraylist "^2.0.0" -it-pipe@^2.0.3, it-pipe@^2.0.4, it-pipe@^2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/it-pipe/-/it-pipe-2.0.5.tgz#9caf7993dcbbc3824bc6ef64ee8b94574f65afa7" - integrity sha512-y85nW1N6zoiTnkidr2EAyC+ZVzc7Mwt2p+xt2a2ooG1ThFakSpNw1Kxm+7F13Aivru96brJhjQVRQNU+w0yozw== - dependencies: - it-merge "^2.0.0" - it-pushable "^3.1.0" - it-stream-types "^1.0.3" +it-peekable@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/it-peekable/-/it-peekable-3.0.1.tgz#530953f735359c10503e961c059602f8a366a1a5" + integrity sha512-5zBfkf6e+YoxxWV0YDXMwdQKnc7eeTX6xo3WYPm/8dIoctIiDnddInRWOW+83W/8/76sbnpWqqsO4gSyXandeQ== -it-pushable@^3.0.0, it-pushable@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/it-pushable/-/it-pushable-3.1.0.tgz#2fba7aaca189595e64e042ac947c6748ece2eb6b" - integrity sha512-sEAdT86u6aIWvLkH4hlOmgvHpRyUOUG22HD365H+Dh67zYpaPdILmT4Om7Wjdb+m/SjEB81z3nYCoIrgVYpOFA== +it-pipe@^3.0.0, it-pipe@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/it-pipe/-/it-pipe-3.0.1.tgz#b25720df82f4c558a8532602b5fbc37bbe4e7ba5" + integrity sha512-sIoNrQl1qSRg2seYSBH/3QxWhJFn9PKYvOf/bHdtCBF0bnghey44VyASsWzn5dAx0DCDDABq1hZIuzKmtBZmKA== + dependencies: + it-merge "^3.0.0" + it-pushable "^3.1.2" + it-stream-types "^2.0.1" -it-pushable@^3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/it-pushable/-/it-pushable-3.1.2.tgz#6f2420fb192f637613c561720945a36b6d9160ae" - integrity sha512-zU9FbeoGT0f+yobwm8agol2OTMXbq4ZSWLEi7hug6TEZx4qVhGhGyp31cayH04aBYsIoO2Nr5kgMjH/oWj2BJQ== +it-pushable@^3.0.0, it-pushable@^3.1.0, it-pushable@^3.1.2, it-pushable@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/it-pushable/-/it-pushable-3.1.3.tgz#b6f4a1e0236502f12b5661b40468b629799baf0e" + integrity sha512-f50iQ85HISS6DaWCyrqf9QJ6G/kQtKIMf9xZkgZgyOvxEQDfn8OfYcLXXquCqgoLboxQtAW1ZFZyFIAsLHDtJw== it-reader@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/it-reader/-/it-reader-6.0.1.tgz#ef7bf7b327cd1f418abb9525641c71658eee21c1" - integrity sha512-C+YRk3OTufbKSJMNEonfEw+9F38llmwwZvqhkjb0xIgob7l4L3p01Yt43+bHRI8SSppAOgk5AKLqas7ea0UTAw== + version "6.0.4" + resolved "https://registry.yarnpkg.com/it-reader/-/it-reader-6.0.4.tgz#439cb88225dcd15116be0ffde9e846a928c3871a" + integrity sha512-XCWifEcNFFjjBHtor4Sfaj8rcpt+FkY0L6WdhD578SCDhV4VUm7fCkF3dv5a+fTcfQqvN9BsxBTvWbYO6iCjTg== dependencies: - it-stream-types "^1.0.4" + it-stream-types "^2.0.1" uint8arraylist "^2.0.0" -it-sort@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/it-sort/-/it-sort-2.0.0.tgz#86b125847a72efad41c274b2a13263e2925af1cc" - integrity sha512-yeAE97b5PEjCrWFUiNyR90eJdGslj8FB3cjT84rsc+mzx9lxPyR2zJkYB9ZOJoWE5MMebxqcQCLRT3OSlzo7Zg== +it-sort@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/it-sort/-/it-sort-3.0.2.tgz#5bf8549b74c74aee20636184791941413b09abf7" + integrity sha512-gRvHyXkn13hyXIoiGkvg7Mf1Yg8JUB+ArKjMrGCYfd/4MQ8mQlMCOE6H8itjshwdVEAUDrppb786zODndYyjSg== dependencies: - it-all "^2.0.0" - -it-stream-types@^1.0.3, it-stream-types@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/it-stream-types/-/it-stream-types-1.0.4.tgz#6e66a11abfd98abab4894c30da15829a0a56bb43" - integrity sha512-0F3CqTIcIHwtnmIgqd03a7sw8BegAmE32N2w7anIGdALea4oAN4ltqPgDMZ7zn4XPLZifXEZlBXSzgg64L1Ebw== + it-all "^3.0.0" -it-stream-types@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/it-stream-types/-/it-stream-types-1.0.5.tgz#9c72e6adefdea9dac69d0a28fbea783deebd508d" - integrity sha512-I88Ka1nHgfX62e5mi5LLL+oueqz7Ltg0bUdtsUKDe9SoUqbQPf2Mp5kxDTe9pNhHQGs4pvYPAINwuZ1HAt42TA== +it-stream-types@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/it-stream-types/-/it-stream-types-2.0.1.tgz#69cb4d7e79e707b8257a8997e02751ccb6c3af32" + integrity sha512-6DmOs5r7ERDbvS4q8yLKENcj6Yecr7QQTqWApbZdfAUTEC947d+PEha7PCqhm//9oxaLYL7TWRekwhoXl2s6fg== -it-take@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/it-take/-/it-take-2.0.0.tgz#e62bdf0f9bf1590b369a116b37de9f74b1f61f00" - integrity sha512-lN3diSTomOvYBk2K0LHAgrQ52DlQfvq8tH/+HLAFpX8Q3JwBkr/BPJEi3Z3Lf8jMmN1KOCBXvt5sXa3eW9vUmg== +it-take@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/it-take/-/it-take-3.0.2.tgz#ba947c6300a36556e223b4f5ab0bffba4b4fbbb1" + integrity sha512-HgtnQYW45iV+lOJIk54dhKWNi+puAeutUehIWQE9tRkM91nlCn0abbDU2xG/FZV3cVnEG4hGwxOEImnMMKwhmg== jake@^10.8.5: version "10.8.5" @@ -10236,75 +10071,71 @@ libnpmpublish@6.0.4: semver "^7.3.7" ssri "^9.0.0" -libp2p@0.42.2: - version "0.42.2" - resolved "https://registry.yarnpkg.com/libp2p/-/libp2p-0.42.2.tgz#093b694b550508fadd8d3bcbd5d42cc984409d0f" - integrity sha512-arTOCJEEmAFw5HjlXdULVAFs7Y/dWZmgX/qN4SzuxtSkB0pa+fqn/DIbIfpBi2BuY+QozvnARPF1xJtSdqfqJQ== - dependencies: - "@achingbrain/nat-port-mapper" "^1.0.3" - "@libp2p/crypto" "^1.0.4" - "@libp2p/interface-address-manager" "^2.0.0" - "@libp2p/interface-connection" "^3.0.2" - "@libp2p/interface-connection-encrypter" "^3.0.1" - "@libp2p/interface-connection-manager" "^1.1.1" - "@libp2p/interface-content-routing" "^2.0.0" - "@libp2p/interface-dht" "^2.0.0" - "@libp2p/interface-libp2p" "^1.0.0" +libp2p@0.45.9: + version "0.45.9" + resolved "https://registry.yarnpkg.com/libp2p/-/libp2p-0.45.9.tgz#cabc2ec8c90ff7fbf07fc0bef2380ff72a2a65c3" + integrity sha512-cf2dCf8naZqQoDw3xxSEZ6rKgQ8BBne5iWgtIKHAYrCvL+ulshz72jNgeAG0FQ/jjRD3yzmUuwoMaLHj6gf7Bw== + dependencies: + "@achingbrain/nat-port-mapper" "^1.0.9" + "@libp2p/crypto" "^1.0.17" + "@libp2p/interface-address-manager" "^3.0.0" + "@libp2p/interface-connection" "^5.1.1" + "@libp2p/interface-connection-encrypter" "^4.0.0" + "@libp2p/interface-connection-gater" "^3.0.0" + "@libp2p/interface-connection-manager" "^3.0.0" + "@libp2p/interface-content-routing" "^2.1.0" + "@libp2p/interface-keychain" "^2.0.4" + "@libp2p/interface-libp2p" "^3.2.0" "@libp2p/interface-metrics" "^4.0.0" - "@libp2p/interface-peer-discovery" "^1.0.1" - "@libp2p/interface-peer-id" "^2.0.0" + "@libp2p/interface-peer-discovery" "^2.0.0" + "@libp2p/interface-peer-id" "^2.0.1" "@libp2p/interface-peer-info" "^1.0.3" - "@libp2p/interface-peer-routing" "^1.0.1" - "@libp2p/interface-peer-store" "^1.2.2" - "@libp2p/interface-pubsub" "^3.0.0" + "@libp2p/interface-peer-routing" "^1.1.0" + "@libp2p/interface-peer-store" "^2.0.4" + "@libp2p/interface-pubsub" "^4.0.0" + "@libp2p/interface-record" "^2.0.6" "@libp2p/interface-registrar" "^2.0.3" - "@libp2p/interface-stream-muxer" "^3.0.0" - "@libp2p/interface-transport" "^2.1.0" - "@libp2p/interfaces" "^3.0.3" - "@libp2p/logger" "^2.0.1" - "@libp2p/multistream-select" "^3.0.0" + "@libp2p/interface-stream-muxer" "^4.0.0" + "@libp2p/interface-transport" "^4.0.0" + "@libp2p/interfaces" "^3.2.0" + "@libp2p/keychain" "^2.0.0" + "@libp2p/logger" "^2.1.1" + "@libp2p/multistream-select" "^3.1.8" "@libp2p/peer-collections" "^3.0.0" "@libp2p/peer-id" "^2.0.0" "@libp2p/peer-id-factory" "^2.0.0" "@libp2p/peer-record" "^5.0.0" - "@libp2p/peer-store" "^6.0.0" + "@libp2p/peer-store" "^8.2.0" + "@libp2p/topology" "^4.0.1" "@libp2p/tracked-map" "^3.0.0" - "@libp2p/utils" "^3.0.2" - "@multiformats/mafmt" "^11.0.2" - "@multiformats/multiaddr" "^11.0.0" - abortable-iterator "^4.0.2" - any-signal "^3.0.0" - datastore-core "^8.0.1" - err-code "^3.0.1" - events "^3.3.0" - hashlru "^2.3.0" - interface-datastore "^7.0.0" - it-all "^2.0.0" - it-drain "^2.0.0" - it-filter "^2.0.0" - it-first "^2.0.0" - it-foreach "^1.0.0" - it-handshake "^4.1.2" - it-length-prefixed "^8.0.2" - it-map "^2.0.0" - it-merge "^2.0.0" + "@libp2p/utils" "^3.0.10" + "@multiformats/mafmt" "^12.0.0" + "@multiformats/multiaddr" "^12.0.0" + abortable-iterator "^5.0.1" + any-signal "^4.1.1" + datastore-core "^9.0.0" + interface-datastore "^8.0.0" + it-all "^3.0.1" + it-drain "^3.0.1" + it-filter "^3.0.1" + it-first "^3.0.1" + it-handshake "^4.1.3" + it-length-prefixed "^9.0.1" + it-map "^3.0.2" + it-merge "^3.0.0" it-pair "^2.0.2" - it-pipe "^2.0.3" - it-sort "^2.0.0" - it-stream-types "^1.0.4" + it-parallel "^3.0.0" + it-pb-stream "^4.0.1" + it-pipe "^3.0.1" + it-stream-types "^2.0.1" merge-options "^3.0.4" multiformats "^11.0.0" - node-forge "^1.3.1" - p-fifo "^1.0.0" + p-defer "^4.0.0" + p-queue "^7.3.4" p-retry "^5.0.0" - p-settle "^5.0.0" private-ip "^3.0.0" - protons-runtime "^4.0.1" + protons-runtime "^5.0.0" rate-limiter-flexible "^2.3.11" - retimer "^3.0.0" - sanitize-filename "^1.6.3" - set-delayed-interval "^1.0.0" - timeout-abort-controller "^3.0.0" uint8arraylist "^2.3.2" uint8arrays "^4.0.2" wherearewe "^2.0.0" @@ -11083,7 +10914,7 @@ mkdirp@*: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-2.1.3.tgz#b083ff37be046fd3d6552468c1f0ff44c1545d1f" integrity sha512-sjAkg21peAG9HS+Dkx7hlG9Ztx7HLeKnvB3NQRcu/mltCVmvkF0pisbiTSfDVYTT86XEfZrTUosLdZLStquZUw== -mkdirp@^0.5.1, mkdirp@^0.5.5: +mkdirp@^0.5.5: version "0.5.6" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== @@ -11154,15 +10985,15 @@ moment@^2.29.1: resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== -mortice@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/mortice/-/mortice-3.0.0.tgz#41a31dd00c799c1d456223d1ca211557316383d4" - integrity sha512-g4rgq//2PWn4m52G6TpCSGmtWabJM8LKCZTQY4W7z0foiaQkqw+FG9a6pwIqUcTkCgBQoet8G/24V6adVMpnHw== +mortice@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/mortice/-/mortice-3.0.1.tgz#27c1943b1841502c7b27a9c8fea789f87c124515" + integrity sha512-eyDUsl1nCR9+JtNksKnaESLP9MgAXCA4w1LTtsmOSQNsThnv++f36rrBu5fC/fdGIwTJZmbiaR/QewptH93pYA== dependencies: - nanoid "^3.1.20" + nanoid "^4.0.0" observable-webworkers "^2.0.1" p-queue "^7.2.0" - p-timeout "^5.0.2" + p-timeout "^6.0.0" ms@2.0.0: version "2.0.0" @@ -11195,12 +11026,12 @@ multibase@~0.6.0: base-x "^3.0.8" buffer "^5.5.0" -multicast-dns@^7.2.0: - version "7.2.2" - resolved "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.2.tgz" - integrity sha512-XqSMeO8EWV/nOXOpPV8ztIpNweVfE1dSpz6SQvDPp71HD74lMXjt4m/mWB1YBMG0kHtOodxRWc5WOb/UNN1A5g== +multicast-dns@^7.2.5: + version "7.2.5" + resolved "https://registry.yarnpkg.com/multicast-dns/-/multicast-dns-7.2.5.tgz#77eb46057f4d7adbd16d9290fa7299f6fa64cced" + integrity sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg== dependencies: - dns-packet "^4.0.0" + dns-packet "^5.2.2" thunky "^1.0.2" multicodec@^0.5.5: @@ -11228,6 +11059,11 @@ multiformats@^11.0.0: resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-11.0.1.tgz#ba58c3f69f032ab67dab4b48cc70f01ac2ca07fe" integrity sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA== +multiformats@^11.0.2: + version "11.0.2" + resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-11.0.2.tgz#b14735efc42cd8581e73895e66bebb9752151b60" + integrity sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg== + multihashes@^0.4.15, multihashes@~0.4.15: version "0.4.21" resolved "https://registry.yarnpkg.com/multihashes/-/multihashes-0.4.21.tgz#dc02d525579f334a7909ade8a122dabb58ccfcb5" @@ -11258,7 +11094,7 @@ mute-stream@1.0.0: resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-1.0.0.tgz#e31bd9fe62f0aed23520aa4324ea6671531e013e" integrity sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA== -nan@^2.13.2, nan@^2.16.0, nan@^2.17.0: +nan@^2.16.0, nan@^2.17.0: version "2.17.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.17.0.tgz#c0150a2368a182f033e9aa5195ec76ea41a199cb" integrity sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ== @@ -11268,7 +11104,7 @@ nano-json-stream-parser@^0.1.2: resolved "https://registry.yarnpkg.com/nano-json-stream-parser/-/nano-json-stream-parser-0.1.2.tgz#0cc8f6d0e2b622b479c40d499c46d64b755c6f5f" integrity sha512-9MqxMH/BSJC7dnLsEMPyfN5Dvoo49IsPFYMcHw3Bcfc2kN0lpHRBSzlMSVx4HGyJ7s9B31CyBTVehWJoQ8Ctew== -nanoid@3.3.3, nanoid@^3.1.20: +nanoid@3.3.3: version "3.3.3" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.3.tgz#fd8e8b7aa761fe807dba2d1b98fb7241bb724a25" integrity sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w== @@ -11298,15 +11134,6 @@ natural-compare@^1.4.0: resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -needle@^2.2.1: - version "2.9.1" - resolved "https://registry.yarnpkg.com/needle/-/needle-2.9.1.tgz#22d1dffbe3490c2b83e301f7709b6736cd8f2684" - integrity sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ== - dependencies: - debug "^3.2.6" - iconv-lite "^0.4.4" - sax "^1.2.4" - negotiator@0.6.3, negotiator@^0.6.2, negotiator@^0.6.3: version "0.6.3" resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz" @@ -11370,7 +11197,7 @@ node-fetch@^2.6.9: dependencies: whatwg-url "^5.0.0" -node-forge@^1.1.0, node-forge@^1.3.1: +node-forge@^1.1.0: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== @@ -11462,22 +11289,6 @@ node-libs-browser@^2.1.0: util "^0.11.0" vm-browserify "^1.0.1" -node-pre-gyp@^0.13.0: - version "0.13.0" - resolved "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.13.0.tgz" - integrity sha512-Md1D3xnEne8b/HGVQkZZwV27WUi1ZRuZBij24TNaZwUPU3ZAFtvT6xxJGaUVillfmMKnn5oD1HoGsp2Ftik7SQ== - dependencies: - detect-libc "^1.0.2" - mkdirp "^0.5.1" - needle "^2.2.1" - nopt "^4.0.1" - npm-packlist "^1.1.6" - npmlog "^4.0.2" - rc "^1.2.7" - rimraf "^2.6.1" - semver "^5.3.0" - tar "^4" - node-preload@^0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz" @@ -11490,14 +11301,6 @@ node-releases@^2.0.6: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.6.tgz#8a7088c63a55e493845683ebf3c828d8c51c5503" integrity sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg== -nopt@^4.0.1: - version "4.0.3" - resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48" - integrity sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg== - dependencies: - abbrev "1" - osenv "^0.1.4" - nopt@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/nopt/-/nopt-5.0.0.tgz#530942bb58a512fccafe53fe210f13a25355dc88" @@ -11574,7 +11377,7 @@ normalize-url@^6.0.1: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== -npm-bundled@^1.0.1, npm-bundled@^1.1.1, npm-bundled@^1.1.2: +npm-bundled@^1.1.1, npm-bundled@^1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz" integrity sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ== @@ -11671,15 +11474,6 @@ npm-packlist@5.1.1: npm-bundled "^1.1.2" npm-normalize-package-bin "^1.0.1" -npm-packlist@^1.1.6: - version "1.4.8" - resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.4.8.tgz#56ee6cc135b9f98ad3d51c1c95da22bbb9b2ef3e" - integrity sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A== - dependencies: - ignore-walk "^3.0.1" - npm-bundled "^1.0.1" - npm-normalize-package-bin "^1.0.1" - npm-packlist@^5.1.0: version "5.1.3" resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-5.1.3.tgz#69d253e6fd664b9058b85005905012e00e69274b" @@ -11788,16 +11582,6 @@ npmlog@6.0.2, npmlog@^6.0.2: gauge "^4.0.3" set-blocking "^2.0.0" -npmlog@^4.0.2: - version "4.1.2" - resolved "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz" - integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg== - dependencies: - are-we-there-yet "~1.1.2" - console-control-strings "~1.1.0" - gauge "~2.7.3" - set-blocking "~2.0.0" - npmlog@^6.0.0: version "6.0.1" resolved "https://registry.npmjs.org/npmlog/-/npmlog-6.0.1.tgz" @@ -11818,11 +11602,6 @@ npmlog@^7.0.1: gauge "^5.0.0" set-blocking "^2.0.0" -number-is-nan@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" - integrity sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ== - number-to-bn@1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/number-to-bn/-/number-to-bn-1.7.0.tgz#bb3623592f7e5f9e0030b1977bd41a0c53fe1ea0" @@ -12095,24 +11874,11 @@ os-browserify@^0.3.0: resolved "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz" integrity sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc= -os-homedir@^1.0.0: - version "1.0.2" - resolved "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz" - integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= - -os-tmpdir@^1.0.0, os-tmpdir@~1.0.2: +os-tmpdir@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" integrity sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g== -osenv@^0.1.4: - version "0.1.5" - resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" - integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g== - dependencies: - os-homedir "^1.0.0" - os-tmpdir "^1.0.0" - p-cancelable@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc" @@ -12128,24 +11894,11 @@ p-cancelable@^3.0.0: resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-3.0.0.tgz#63826694b54d61ca1c20ebcb6d3ecf5e14cd8050" integrity sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw== -p-defer@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz" - integrity sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw== - p-defer@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/p-defer/-/p-defer-4.0.0.tgz#8082770aeeb10eb6b408abe91866738741ddd5d2" integrity sha512-Vb3QRvQ0Y5XnF40ZUWW7JfLogicVh/EnA5gBIvKDJoYpeI82+1E3AlB9yOcKFS0AhHrWVnAQO39fbR0G99IVEQ== -p-fifo@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/p-fifo/-/p-fifo-1.0.0.tgz" - integrity sha512-IjoCxXW48tqdtDFz6fqo5q1UfFVjjVZe8TC1QRflvNUJtNfCUhxOUw6MOVZhDPjqhSzc26xKdugsO17gmzd5+A== - dependencies: - fast-fifo "^1.0.0" - p-defer "^3.0.0" - p-finally@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" @@ -12247,16 +12000,19 @@ p-queue@^7.2.0: eventemitter3 "^4.0.7" p-timeout "^5.0.2" +p-queue@^7.3.4: + version "7.3.4" + resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-7.3.4.tgz#7ef7d89b6c1a0563596d98adbc9dc404e9ed4a84" + integrity sha512-esox8CWt0j9EZECFvkFl2WNPat8LN4t7WWeXq73D9ha0V96qPRufApZi4ZhPwXAln1uVVal429HVVKPa2X0yQg== + dependencies: + eventemitter3 "^4.0.7" + p-timeout "^5.0.2" + p-reduce@2.1.0, p-reduce@^2.0.0, p-reduce@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-reduce/-/p-reduce-2.1.0.tgz#09408da49507c6c274faa31f28df334bc712b64a" integrity sha512-2USApvnsutq8uoxZBGbbWM0JIYLiEMJ9RlaN7fAzVNb9OZN0SHjjTTfIcb667XynS5Y1VhwDJVDa72TnPzAYWw== -p-reflect@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/p-reflect/-/p-reflect-3.0.0.tgz#2473a6f9ee0376a27723b18efd24911db8e92573" - integrity sha512-rOgYyrvUxnJdSYKGSK7UnO7RxFSnT/IJYFPiosuQ2/AtRWIryIrv8lecWqJXWbKnMcUjJvxiHDMp80m0Yj4eLA== - p-retry@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-5.1.1.tgz#1950b9be441474a67f852811c1d4ec955885d2c8" @@ -12265,14 +12021,6 @@ p-retry@^5.0.0: "@types/retry" "0.12.1" retry "^0.13.1" -p-settle@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/p-settle/-/p-settle-5.0.0.tgz#b7008de2f225ed9132317d995ead4c007684024e" - integrity sha512-P+cL1wECSDqI49JAiHlgG0HlqqL0CBsMP3f8vrVx6Yy8pMngmJqn8UjzAHr5CPkcDIzeBxugDLDMWTK8fqrFGw== - dependencies: - p-limit "^4.0.0" - p-reflect "^3.0.0" - p-timeout@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" @@ -12285,6 +12033,16 @@ p-timeout@^5.0.2: resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-5.1.0.tgz#b3c691cf4415138ce2d9cfe071dba11f0fee085b" integrity sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew== +p-timeout@^6.0.0: + version "6.1.1" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-6.1.1.tgz#bcee5e37d730f5474d973b6ff226751a1a5e6ff1" + integrity sha512-yqz2Wi4fiFRpMmK0L2pGAU49naSUaP23fFIQL2Y6YT+qDGPoFwpvgQM/wzc6F8JoenUkIlAFa4Ql7NguXBxI7w== + +p-timeout@^6.1.1: + version "6.1.2" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-6.1.2.tgz#22b8d8a78abf5e103030211c5fc6dee1166a6aa5" + integrity sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ== + p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" @@ -12666,16 +12424,6 @@ pretty-format@29.4.3: ansi-styles "^5.0.0" react-is "^18.0.0" -private-ip@^2.1.1: - version "2.3.4" - resolved "https://registry.yarnpkg.com/private-ip/-/private-ip-2.3.4.tgz#e2944f2a7a0142ec6640efda323af4b96307524e" - integrity sha512-ts/YFVwfBeLq61f9+KsOhXW6RH0wvY0gU50R6QZYzgFhggyyLK6WDFeYdjfi/HMnBm2hecLvsR3PB3JcRxDk+A== - dependencies: - ip-regex "^4.3.0" - ipaddr.js "^2.0.1" - is-ip "^3.1.0" - netmask "^2.0.2" - private-ip@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/private-ip/-/private-ip-3.0.0.tgz#a65d10e2db06f6bb2f97f716f1a8976a3460a4a6" @@ -12815,14 +12563,6 @@ protocols@^2.0.0, protocols@^2.0.1: resolved "https://registry.yarnpkg.com/protocols/-/protocols-2.0.1.tgz#8f155da3fc0f32644e83c5782c8e8212ccf70a86" integrity sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q== -protons-runtime@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/protons-runtime/-/protons-runtime-4.0.1.tgz#bcea3667b6263680e70e2da102f91b3513075374" - integrity sha512-SPeV+8TzJAp5UJYPV7vJkLRi08CP0DksxpKK60rcNaZSPkMBQwc0jQrmkHqwc5P0cYbZzKsdYrUBwRrDLrzTfQ== - dependencies: - protobufjs "^7.0.0" - uint8arraylist "^2.3.2" - protons-runtime@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/protons-runtime/-/protons-runtime-5.0.0.tgz#1eb3c78637ff02cc90bb030e3bff6f0402109c25" @@ -13024,16 +12764,6 @@ raw-body@2.5.2: iconv-lite "0.4.24" unpipe "1.0.0" -rc@^1.2.7: - version "1.2.8" - resolved "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz" - integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== - dependencies: - deep-extend "^0.6.0" - ini "~1.3.0" - minimist "^1.2.0" - strip-json-comments "~2.0.1" - react-is@^18.0.0: version "18.2.0" resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b" @@ -13147,7 +12877,7 @@ readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.2, readable-stre string_decoder "^1.1.1" util-deprecate "^1.0.1" -readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@^2.3.7, readable-stream@~2.3.6: +readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@^2.3.7, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -13371,11 +13101,6 @@ ret@~0.2.0: resolved "https://registry.npmjs.org/ret/-/ret-0.2.2.tgz" integrity sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ== -retimer@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/retimer/-/retimer-3.0.0.tgz" - integrity sha512-WKE0j11Pa0ZJI5YIk0nflGI7SQsfl2ljihVy7ogh7DeQSeYAUi0ubZ/yEueGtDfUPk6GH5LRw1hBdLq4IwUBWA== - retry@0.13.1, retry@^0.13.1: version "0.13.1" resolved "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz" @@ -13410,13 +13135,6 @@ rfdc@^1.2.0, rfdc@^1.3.0: resolved "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz" integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA== -rimraf@^2.6.1: - version "2.7.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" - integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== - dependencies: - glob "^7.1.3" - rimraf@^3.0.0, rimraf@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" @@ -13558,7 +13276,7 @@ sax@1.2.1: resolved "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz" integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= -sax@>=0.6.0, sax@^1.2.4: +sax@>=0.6.0: version "1.2.4" resolved "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== @@ -13605,7 +13323,7 @@ semver-compare@^1.0.0: resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" integrity sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow== -"semver@2 || 3 || 4 || 5", semver@^5.3.0, semver@^5.4.1, semver@^5.5.0, semver@^5.6.0: +"semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.5.0, semver@^5.6.0: version "5.7.1" resolved "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== @@ -13704,7 +13422,7 @@ servify@^0.1.12: request "^2.79.0" xhr "^2.3.3" -set-blocking@^2.0.0, set-blocking@~2.0.0: +set-blocking@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= @@ -13714,11 +13432,6 @@ set-cookie-parser@^2.4.1: resolved "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.4.8.tgz" integrity sha512-edRH8mBKEWNVIVMKejNnuJxleqYE/ZSdcT8/Nem9/mmosx12pctd80s2Oy00KNZzrogMZS5mauK2/ymL1bvlvg== -set-delayed-interval@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/set-delayed-interval/-/set-delayed-interval-1.0.0.tgz" - integrity sha512-29fhAwuZlLcuBnW/EwxvLcg2D3ELX+VBDNhnavs3YYkab72qmrcSeQNVdzl8EcPPahGQXhBM6MKdPLCQGMDakw== - setimmediate@^1.0.4, setimmediate@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz" @@ -13782,7 +13495,7 @@ side-channel@^1.0.4: get-intrinsic "^1.0.2" object-inspect "^1.9.0" -signal-exit@3.0.7, signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7: +signal-exit@3.0.7, signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7: version "3.0.7" resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== @@ -14176,15 +13889,6 @@ strict-uri-encode@^1.0.0: resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713" integrity sha512-R3f198pcvnB+5IpnBlRkphuE9n46WyVl8I39W/ZUTZLz4nqSP/oLYUrcnJrw462Ds8he4YKMov2efsTIw1BDGQ== -string-width@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" - integrity sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw== - dependencies: - code-point-at "^1.0.0" - is-fullwidth-code-point "^1.0.0" - strip-ansi "^3.0.0" - "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" @@ -14271,13 +13975,6 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -strip-ansi@^3.0.0, strip-ansi@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" - integrity sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg== - dependencies: - ansi-regex "^2.0.0" - strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" @@ -14326,7 +14023,7 @@ strip-json-comments@3.1.1, strip-json-comments@^3.1.0, strip-json-comments@^3.1. resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== -strip-json-comments@^2.0.0, strip-json-comments@~2.0.1: +strip-json-comments@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz" integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= @@ -14472,7 +14169,7 @@ tar@6.1.11, tar@^6.0.2, tar@^6.1.11, tar@^6.1.2: mkdirp "^1.0.3" yallist "^4.0.0" -tar@^4, tar@^4.0.2: +tar@^4.0.2: version "4.4.19" resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.19.tgz#2e4d7263df26f2b914dee10c825ab132123742f3" integrity sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA== @@ -14638,13 +14335,6 @@ timed-out@^4.0.1: resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f" integrity sha512-G7r3AhovYtr5YKOWQkta8RKAPb+J9IsO4uVmzjl8AZwfhs8UcUwTiD6gcJYSgOtzyjvQKrKYn41syHbUWMkafA== -timeout-abort-controller@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/timeout-abort-controller/-/timeout-abort-controller-3.0.0.tgz" - integrity sha512-O3e+2B8BKrQxU2YRyEjC/2yFdb33slI22WRdUaDx6rvysfi9anloNZyR2q0l6LnePo5qH7gSM7uZtvvwZbc2yA== - dependencies: - retimer "^3.0.0" - timers-browserify@^2.0.4: version "2.0.12" resolved "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz" @@ -15031,7 +14721,7 @@ uint8-varint@^1.0.6: uint8arraylist "^2.0.0" uint8arrays "^4.0.2" -uint8arraylist@^2.0.0, uint8arraylist@^2.1.0, uint8arraylist@^2.1.1, uint8arraylist@^2.1.2, uint8arraylist@^2.3.1, uint8arraylist@^2.3.2, uint8arraylist@^2.4.1, uint8arraylist@^2.4.3: +uint8arraylist@^2.0.0, uint8arraylist@^2.1.0, uint8arraylist@^2.1.1, uint8arraylist@^2.3.1, uint8arraylist@^2.3.2, uint8arraylist@^2.4.1, uint8arraylist@^2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/uint8arraylist/-/uint8arraylist-2.4.3.tgz#1148aa979b407d382e4eb8d9c8f2b4bf3f5910d5" integrity sha512-oEVZr4/GrH87K0kjNce6z8pSCzLEPqHNLNR5sj8cJOySrTP8Vb/pMIbZKLJGhQKxm1TiZ31atNrpn820Pyqpow== @@ -15748,7 +15438,7 @@ which@^3.0.0: dependencies: isexe "^2.0.0" -wide-align@^1.1.0, wide-align@^1.1.5: +wide-align@^1.1.5: version "1.1.5" resolved "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz" integrity sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg== @@ -15989,6 +15679,14 @@ xml2js@^0.4.19, xml2js@^0.4.23: sax ">=0.6.0" xmlbuilder "~11.0.0" +xml2js@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.0.tgz#07afc447a97d2bd6507a1f76eeadddb09f7a8282" + integrity sha512-eLTh0kA8uHceqesPqSE+VvO1CDDJWMwlQfB6LuN6T8w6MaDJ8Txm8P7s5cHD0miF0V+GGTZrDQfxPZQVsur33w== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + xmlbuilder@~11.0.0: version "11.0.1" resolved "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz" From 1de881b7776284c6dd56f146ac55ef3df040f98d Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 7 Jul 2023 14:13:11 +0200 Subject: [PATCH 47/96] chore: remove explicit exit after beacon node closed (#5735) --- packages/cli/src/cmds/beacon/handler.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/cli/src/cmds/beacon/handler.ts b/packages/cli/src/cmds/beacon/handler.ts index 4c71e4714dd2..af34cf861697 100644 --- a/packages/cli/src/cmds/beacon/handler.ts +++ b/packages/cli/src/cmds/beacon/handler.ts @@ -130,9 +130,6 @@ export async function beaconHandler(args: BeaconArgs & GlobalArgs): Promise Date: Mon, 10 Jul 2023 10:48:00 +0200 Subject: [PATCH 48/96] test(prover): improve test coverage (#5648) --- packages/cli/package.json | 3 +- .../cli/test/e2e/blsToExecutionchange.test.ts | 45 ++- .../cli/test/e2e/importFromFsDirect.test.ts | 38 +- .../cli/test/e2e/importFromFsPreStep.test.ts | 27 +- .../test/e2e/importKeystoresFromApi.test.ts | 43 ++- .../test/e2e/importRemoteKeysFromApi.test.ts | 48 ++- .../e2e/propserConfigfromKeymanager.test.ts | 239 ++++++------ packages/cli/test/e2e/runDevCmd.test.ts | 35 +- packages/cli/test/e2e/validatorList.test.ts | 23 +- packages/cli/test/e2e/voluntaryExit.test.ts | 76 ++-- packages/cli/test/utils/childprocRunner.ts | 146 ------- packages/cli/test/utils/inMemoryRunner.ts | 26 -- .../cli/test/utils/keymanagerTestRunners.ts | 87 ----- packages/cli/test/utils/runUtils.ts | 42 -- packages/cli/test/utils/shell.ts | 50 --- .../simulation/runner/ChildProcessRunner.ts | 32 +- .../utils/simulation/runner/DockerRunner.ts | 62 +-- .../utils/simulation/utils/child_process.ts | 97 ----- packages/cli/test/utils/validator.ts | 102 +++++ packages/prover/package.json | 4 +- packages/prover/src/cli/applyPreset.ts | 81 ++++ packages/prover/src/cli/cmds/start/handler.ts | 8 +- packages/prover/src/cli/index.ts | 1 + packages/prover/src/cli/options.ts | 41 +- packages/prover/src/utils/file.ts | 51 +++ packages/prover/src/web3_proxy.ts | 2 +- .../prover/test/e2e/cli/cmds/start.test.ts | 92 +++++ .../test/e2e/web3_batch_request.test.ts | 4 +- .../prover/test/e2e/web3_provider.test.ts | 6 +- packages/prover/test/utils/e2e_env.ts | 5 +- packages/prover/test/utils/network.ts | 19 - packages/test-utils/.mocharc.yaml | 4 + packages/test-utils/.nycrc.json | 3 + packages/test-utils/LICENSE | 201 ++++++++++ packages/test-utils/README.md | 11 + packages/test-utils/package.json | 79 ++++ packages/test-utils/src/childProcess.ts | 359 ++++++++++++++++++ packages/test-utils/src/cli.ts | 91 +++++ packages/test-utils/src/http.ts | 55 +++ packages/test-utils/src/index.ts | 6 + packages/test-utils/src/interfaces.ts | 5 + packages/test-utils/src/mocha.ts | 45 +++ packages/test-utils/src/path.ts | 21 + packages/test-utils/src/sinon.ts | 23 ++ packages/test-utils/src/timeout.ts | 17 + packages/test-utils/tsconfig.build.json | 8 + packages/test-utils/tsconfig.json | 6 + 47 files changed, 1684 insertions(+), 785 deletions(-) delete mode 100644 packages/cli/test/utils/childprocRunner.ts delete mode 100644 packages/cli/test/utils/inMemoryRunner.ts delete mode 100644 packages/cli/test/utils/keymanagerTestRunners.ts delete mode 100644 packages/cli/test/utils/shell.ts delete mode 100644 packages/cli/test/utils/simulation/utils/child_process.ts create mode 100644 packages/cli/test/utils/validator.ts create mode 100644 packages/prover/src/cli/applyPreset.ts create mode 100644 packages/prover/src/utils/file.ts create mode 100644 packages/prover/test/e2e/cli/cmds/start.test.ts delete mode 100644 packages/prover/test/utils/network.ts create mode 100644 packages/test-utils/.mocharc.yaml create mode 100644 packages/test-utils/.nycrc.json create mode 100644 packages/test-utils/LICENSE create mode 100644 packages/test-utils/README.md create mode 100644 packages/test-utils/package.json create mode 100644 packages/test-utils/src/childProcess.ts create mode 100644 packages/test-utils/src/cli.ts create mode 100644 packages/test-utils/src/http.ts create mode 100644 packages/test-utils/src/index.ts create mode 100644 packages/test-utils/src/interfaces.ts create mode 100644 packages/test-utils/src/mocha.ts create mode 100644 packages/test-utils/src/path.ts create mode 100644 packages/test-utils/src/sinon.ts create mode 100644 packages/test-utils/src/timeout.ts create mode 100644 packages/test-utils/tsconfig.build.json create mode 100644 packages/test-utils/tsconfig.json diff --git a/packages/cli/package.json b/packages/cli/package.json index 5a3873a58316..e51543e563a3 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -99,6 +99,7 @@ "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", "@types/lodash": "^4.14.192", - "@types/yargs": "^17.0.24" + "@types/yargs": "^17.0.24", + "@lodestar/test-utils": "^1.9.1" } } diff --git a/packages/cli/test/e2e/blsToExecutionchange.test.ts b/packages/cli/test/e2e/blsToExecutionchange.test.ts index 67b0a5969c8c..c36fea252db6 100644 --- a/packages/cli/test/e2e/blsToExecutionchange.test.ts +++ b/packages/cli/test/e2e/blsToExecutionchange.test.ts @@ -4,36 +4,41 @@ import {ApiError, getClient} from "@lodestar/api"; import {config} from "@lodestar/config/default"; import {interopSecretKey} from "@lodestar/state-transition"; import {toHexString} from "@chainsafe/ssz"; +import {execCliCommand, spawnCliCommand, stopChildProcess} from "@lodestar/test-utils"; import {testFilesDir} from "../utils.js"; -import {describeCliTest, execCli} from "../utils/childprocRunner.js"; -import {itDone} from "../utils/runUtils.js"; -describeCliTest("bLSToExecutionChange cmd", function ({spawnCli}) { +describe("bLSToExecutionChange cmd", function () { this.timeout("60s"); - itDone("Perform bLSToExecutionChange", async function (done) { + it("Perform bLSToExecutionChange", async () => { const restPort = 9596; - const devBnProc = spawnCli({pipeStdToParent: false, logPrefix: "dev"}, [ - // ⏎ - "dev", - `--dataDir=${path.join(testFilesDir, "dev-bls-to-execution-change")}`, - "--genesisValidators=8", - "--startValidators=0..7", - "--rest", - `--rest.port=${restPort}`, - // Speed up test to make genesis happen faster - "--params.SECONDS_PER_SLOT=2", - ]); + const devBnProc = await spawnCliCommand( + "packages/cli/bin/lodestar.js", + [ + "dev", + `--dataDir=${path.join(testFilesDir, "dev-bls-to-execution-change")}`, + "--genesisValidators=8", + "--startValidators=0..7", + "--rest", + `--rest.port=${restPort}`, + // Speed up test to make genesis happen faster + "--params.SECONDS_PER_SLOT=2", + ], + {pipeStdioToParent: false, logPrefix: "dev"} + ); + // Exit early if process exits devBnProc.on("exit", (code) => { if (code !== null && code > 0) { - done(Error(`devBnProc process exited with code ${code}`)); + throw new Error(`devBnProc process exited with code ${code}`); } }); const baseUrl = `http://127.0.0.1:${restPort}`; - const client = getClient({baseUrl}, {config}); + // To cleanup the event stream connection + const httpClientController = new AbortController(); + const client = getClient({baseUrl, getAbortSignal: () => httpClientController.signal}, {config}); // Wait for beacon node API to be available + genesis await retry( @@ -57,8 +62,7 @@ describeCliTest("bLSToExecutionChange cmd", function ({spawnCli}) { // 2 0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4 // 3 0x88c141df77cd9d8d7a71a75c826c41a9c9f03c6ee1b180f3e7852f6a280099ded351b58d66e653 - await execCli([ - // ⏎ + await execCliCommand("packages/cli/bin/lodestar.js", [ "validator", "bls-to-execution-change", "--network=dev", @@ -80,8 +84,9 @@ describeCliTest("bLSToExecutionChange cmd", function ({spawnCli}) { throw Error("Invalid message generated"); } + httpClientController.abort(); devBnProc.kill("SIGINT"); await sleep(1000); - devBnProc.kill("SIGKILL"); + await stopChildProcess(devBnProc, "SIGKILL"); }); }); diff --git a/packages/cli/test/e2e/importFromFsDirect.test.ts b/packages/cli/test/e2e/importFromFsDirect.test.ts index 828aea030a5e..f55587ced2e3 100644 --- a/packages/cli/test/e2e/importFromFsDirect.test.ts +++ b/packages/cli/test/e2e/importFromFsDirect.test.ts @@ -1,14 +1,16 @@ import fs from "node:fs"; import path from "node:path"; import {rimraf} from "rimraf"; +import {getMochaContext} from "@lodestar/test-utils/mocha"; import {testFilesDir} from "../utils.js"; -import {describeCliTest} from "../utils/childprocRunner.js"; -import {getAfterEachCallbacks} from "../utils/runUtils.js"; import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js"; -import {expectKeys, getKeymanagerTestRunner} from "../utils/keymanagerTestRunners.js"; +import {expectKeys, startValidatorWithKeyManager} from "../utils/validator.js"; import {getKeystoresStr} from "../utils/keystores.js"; -describeCliTest("import from fs same cmd as validate", function ({spawnCli}) { +describe("import from fs same cmd as validate", function () { + const testContext = getMochaContext(this); + this.timeout("30s"); + const dataDir = path.join(testFilesDir, "import-and-validate-test"); const importFromDir = path.join(dataDir, "eth2.0_deposit_out"); const passphraseFilepath = path.join(importFromDir, "password.text"); @@ -18,9 +20,6 @@ describeCliTest("import from fs same cmd as validate", function ({spawnCli}) { rimraf.sync(importFromDir); }); - const afterEachCallbacks = getAfterEachCallbacks(); - const itKeymanagerStep = getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, dataDir}); - const passphrase = "AAAAAAAA0000000000"; const keyCount = 2; const pubkeys = cachedPubkeysHex.slice(0, keyCount); @@ -38,18 +37,23 @@ describeCliTest("import from fs same cmd as validate", function ({spawnCli}) { }); // Check that there are not keys loaded without adding extra args `--importKeystores` - itKeymanagerStep("run 'validator' check keys are loaded", async function (keymanagerClient) { + it("run 'validator' there are no keys loaded", async () => { + const {keymanagerClient} = await startValidatorWithKeyManager([], { + dataDir, + logPrefix: "case-1", + testContext, + }); + await expectKeys(keymanagerClient, [], "Wrong listKeys response data"); }); // Run validator with extra arguments to load keystores in same step - itKeymanagerStep( - "run 'validator' check keys are loaded", - async function (keymanagerClient) { - await expectKeys(keymanagerClient, pubkeys, "Wrong listKeys response data"); - }, - { - validatorCmdExtraArgs: [`--importKeystores=${importFromDir}`, `--importKeystoresPassword=${passphraseFilepath}`], - } - ); + it("run 'validator' check keys are loaded", async () => { + const {keymanagerClient} = await startValidatorWithKeyManager( + [`--importKeystores=${importFromDir}`, `--importKeystoresPassword=${passphraseFilepath}`], + {dataDir, logPrefix: "case-2", testContext} + ); + + await expectKeys(keymanagerClient, pubkeys, "Wrong listKeys response data"); + }); }); diff --git a/packages/cli/test/e2e/importFromFsPreStep.test.ts b/packages/cli/test/e2e/importFromFsPreStep.test.ts index 36af4e454934..9dd48acaa1a6 100644 --- a/packages/cli/test/e2e/importFromFsPreStep.test.ts +++ b/packages/cli/test/e2e/importFromFsPreStep.test.ts @@ -2,14 +2,17 @@ import fs from "node:fs"; import path from "node:path"; import {rimraf} from "rimraf"; import {expect} from "chai"; +import {getMochaContext} from "@lodestar/test-utils/mocha"; +import {execCliCommand} from "@lodestar/test-utils"; import {testFilesDir} from "../utils.js"; -import {describeCliTest, execCli} from "../utils/childprocRunner.js"; -import {getAfterEachCallbacks} from "../utils/runUtils.js"; import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js"; -import {expectKeys, getKeymanagerTestRunner} from "../utils/keymanagerTestRunners.js"; +import {expectKeys, startValidatorWithKeyManager} from "../utils/validator.js"; import {getKeystoresStr} from "../utils/keystores.js"; -describeCliTest("import from fs then validate", function ({spawnCli}) { +describe("import from fs then validate", function () { + const testContext = getMochaContext(this); + this.timeout("30s"); + const dataDir = path.join(testFilesDir, "import-then-validate-test"); const importFromDir = path.join(dataDir, "eth2.0_deposit_out"); const passphraseFilepath = path.join(importFromDir, "password.text"); @@ -19,9 +22,6 @@ describeCliTest("import from fs then validate", function ({spawnCli}) { rimraf.sync(importFromDir); }); - const afterEachCallbacks = getAfterEachCallbacks(); - const itKeymanagerStep = getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, dataDir}); - const passphrase = "AAAAAAAA0000000000"; const keyCount = 2; const pubkeys = cachedPubkeysHex.slice(0, keyCount); @@ -37,8 +37,7 @@ describeCliTest("import from fs then validate", function ({spawnCli}) { fs.writeFileSync(path.join(importFromDir, `keystore_${i}.json`), keystoresStr[i]); } - const stdout = await execCli([ - // ⏎ + const stdout = await execCliCommand("packages/cli/bin/lodestar.js", [ "validator import", `--dataDir ${dataDir}`, `--importKeystores ${importFromDir}`, @@ -54,18 +53,16 @@ describeCliTest("import from fs then validate", function ({spawnCli}) { fs.mkdirSync(path.join(dataDir, "keystores"), {recursive: true}); fs.mkdirSync(path.join(dataDir, "secrets"), {recursive: true}); - const stdout = await execCli([ - // ⏎ - "validator list", - `--dataDir ${dataDir}`, - ]); + const stdout = await execCliCommand("packages/cli/bin/lodestar.js", ["validator list", `--dataDir ${dataDir}`]); for (let i = 0; i < keyCount; i++) { expect(stdout).includes(pubkeys[i], `stdout should include imported pubkey[${i}]`); } }); - itKeymanagerStep("run 'validator' check keys are loaded", async function (keymanagerClient) { + it("run 'validator' check keys are loaded", async function () { + const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext}); + await expectKeys(keymanagerClient, pubkeys, "Wrong listKeys response data"); }); }); diff --git a/packages/cli/test/e2e/importKeystoresFromApi.test.ts b/packages/cli/test/e2e/importKeystoresFromApi.test.ts index f88d0f4e55c8..d7bd90033c90 100644 --- a/packages/cli/test/e2e/importKeystoresFromApi.test.ts +++ b/packages/cli/test/e2e/importKeystoresFromApi.test.ts @@ -5,23 +5,24 @@ import {DeletionStatus, getClient, ImportStatus} from "@lodestar/api/keymanager" import {config} from "@lodestar/config/default"; import {Interchange} from "@lodestar/validator"; import {ApiError, HttpStatusCode} from "@lodestar/api"; +import {bufferStderr, spawnCliCommand} from "@lodestar/test-utils"; +import {getMochaContext} from "@lodestar/test-utils/mocha"; import {testFilesDir} from "../utils.js"; -import {bufferStderr, describeCliTest} from "../utils/childprocRunner.js"; import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js"; -import {expectDeepEquals, getAfterEachCallbacks} from "../utils/runUtils.js"; -import {expectKeys, getKeymanagerTestRunner} from "../utils/keymanagerTestRunners.js"; +import {expectDeepEquals} from "../utils/runUtils.js"; +import {expectKeys, startValidatorWithKeyManager} from "../utils/validator.js"; import {getKeystoresStr} from "../utils/keystores.js"; -describeCliTest("import keystores from api", function ({spawnCli}) { +describe("import keystores from api", function () { + const testContext = getMochaContext(this); + this.timeout("30s"); + const dataDir = path.join(testFilesDir, "import-keystores-test"); before("Clean dataDir", () => { rimraf.sync(dataDir); }); - const afterEachCallbacks = getAfterEachCallbacks(); - const itKeymanagerStep = getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, dataDir}); - /** Generated from const sk = bls.SecretKey.fromKeygen(Buffer.alloc(32, 0xaa)); */ const passphrase = "AAAAAAAA0000000000"; const keyCount = 2; @@ -55,7 +56,8 @@ describeCliTest("import keystores from api", function ({spawnCli}) { const slashingProtectionStr = JSON.stringify(slashingProtection); - itKeymanagerStep("run 'validator' and import remote keys from API", async function (keymanagerClient) { + it("run 'validator' and import remote keys from API", async () => { + const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext}); // Produce and encrypt keystores const keystoresStr = await getKeystoresStr(passphrase, secretKeys); @@ -84,16 +86,14 @@ describeCliTest("import keystores from api", function ({spawnCli}) { ); // Attempt to run a second process and expect the keystore lock to throw - const vcProc2 = spawnCli({pipeStdToParent: true, logPrefix: "vc-2"}, [ - // ⏎ - "validator", - `--dataDir=${dataDir}`, - ]); + const validator = await spawnCliCommand("packages/cli/bin/lodestar.js", ["validator", "--dataDir", dataDir], { + logPrefix: "vc-2", + }); await new Promise((resolve, reject) => { // logger.error is printed to stdout, Yargs errors are printed in stderr - const vcProc2Stderr = bufferStderr(vcProc2); - vcProc2.on("exit", (code) => { + const vcProc2Stderr = bufferStderr(validator); + validator.on("exit", (code) => { if (code !== null && code > 0) { // process should exit with code > 0, and an error related to locks. Sample error: // vc 351591: ✖ Error: EEXIST: file already exists, open '/tmp/tmp-351554-dMctEAj7sJIz/import-keystores-test/keystores/0x8be678633e927aa0435addad5dcd5283fef6110d91362519cd6d43e61f6c017d724fa579cc4b2972134e050b6ba120c0/voting-keystore.json.lock' @@ -111,7 +111,9 @@ describeCliTest("import keystores from api", function ({spawnCli}) { }); }); - itKeymanagerStep("run 'validator' check keys are loaded + delete", async function (keymanagerClient) { + it("run 'validator' check keys are loaded + delete", async function () { + const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext}); + // Check that keys imported in previous it() are still there await expectKeys(keymanagerClient, pubkeys, "Wrong listKeys before deleting"); @@ -128,13 +130,16 @@ describeCliTest("import keystores from api", function ({spawnCli}) { await expectKeys(keymanagerClient, [], "Wrong listKeys after deleting"); }); - itKeymanagerStep("different process check no keys are loaded", async function (keymanagerClient) { + it("different process check no keys are loaded", async function () { + const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext}); // After deleting there should be no keys await expectKeys(keymanagerClient, [], "Wrong listKeys"); }); - itKeymanagerStep("reject calls without bearerToken", async function (_, {keymanagerUrl}) { - const keymanagerClientNoAuth = getClient({baseUrl: keymanagerUrl, bearerToken: undefined}, {config}); + it("reject calls without bearerToken", async function () { + await startValidatorWithKeyManager([], {dataDir, testContext}); + + const keymanagerClientNoAuth = getClient({baseUrl: "http://localhost:38011", bearerToken: undefined}, {config}); const res = await keymanagerClientNoAuth.listRemoteKeys(); expect(res.ok).to.be.false; expect(res.error?.code).to.be.eql(HttpStatusCode.UNAUTHORIZED); diff --git a/packages/cli/test/e2e/importRemoteKeysFromApi.test.ts b/packages/cli/test/e2e/importRemoteKeysFromApi.test.ts index bf3de952575f..7f36a6876fd0 100644 --- a/packages/cli/test/e2e/importRemoteKeysFromApi.test.ts +++ b/packages/cli/test/e2e/importRemoteKeysFromApi.test.ts @@ -4,27 +4,40 @@ import {expect} from "chai"; import {Api, DeleteRemoteKeyStatus, getClient, ImportRemoteKeyStatus} from "@lodestar/api/keymanager"; import {config} from "@lodestar/config/default"; import {ApiError, HttpStatusCode} from "@lodestar/api"; +import {getMochaContext} from "@lodestar/test-utils/mocha"; import {testFilesDir} from "../utils.js"; -import {describeCliTest} from "../utils/childprocRunner.js"; import {cachedPubkeysHex} from "../utils/cachedKeys.js"; -import {expectDeepEquals, getAfterEachCallbacks} from "../utils/runUtils.js"; -import {getKeymanagerTestRunner} from "../utils/keymanagerTestRunners.js"; +import {expectDeepEquals} from "../utils/runUtils.js"; +import {startValidatorWithKeyManager} from "../utils/validator.js"; + +const url = "https://remote.signer"; + +async function expectKeys(keymanagerClient: Api, expectedPubkeys: string[], message: string): Promise { + const remoteKeys = await keymanagerClient.listRemoteKeys(); + ApiError.assert(remoteKeys); + expectDeepEquals( + remoteKeys.response.data, + expectedPubkeys.map((pubkey) => ({pubkey, url, readonly: false})), + message + ); +} + +describe("import remoteKeys from api", function () { + const testContext = getMochaContext(this); + this.timeout("30s"); -describeCliTest("import remoteKeys from api", function ({spawnCli}) { const dataDir = path.join(testFilesDir, "import-remoteKeys-test"); before("Clean dataDir", () => { rimraf.sync(dataDir); }); - const afterEachCallbacks = getAfterEachCallbacks(); - const itKeymanagerStep = getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, dataDir}); - /** Generated from const sk = bls.SecretKey.fromKeygen(Buffer.alloc(32, 0xaa)); */ - const url = "https://remote.signer"; const pubkeysToAdd = [cachedPubkeysHex[0], cachedPubkeysHex[1]]; - itKeymanagerStep("run 'validator' and import remote keys from API", async function (keymanagerClient) { + it("run 'validator' and import remote keys from API", async () => { + const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext}); + // Wrap in retry since the API may not be listening yet await expectKeys(keymanagerClient, [], "Wrong listRemoteKeys before importing"); @@ -50,7 +63,8 @@ describeCliTest("import remoteKeys from api", function ({spawnCli}) { ); }); - itKeymanagerStep("run 'validator' check keys are loaded + delete", async function (keymanagerClient) { + it("run 'validator' check keys are loaded + delete", async function () { + const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext}); // Check that keys imported in previous it() are still there await expectKeys(keymanagerClient, pubkeysToAdd, "Wrong listRemoteKeys before deleting"); @@ -67,20 +81,12 @@ describeCliTest("import remoteKeys from api", function ({spawnCli}) { await expectKeys(keymanagerClient, [], "Wrong listRemoteKeys after deleting"); }); - itKeymanagerStep("reject calls without bearerToken", async function (_, {keymanagerUrl}) { + it("reject calls without bearerToken", async function () { + await startValidatorWithKeyManager([], {dataDir, testContext}); + const keymanagerUrl = "http://localhost:38011"; const keymanagerClientNoAuth = getClient({baseUrl: keymanagerUrl, bearerToken: undefined}, {config}); const res = await keymanagerClientNoAuth.listRemoteKeys(); expect(res.ok).to.be.false; expect(res.error?.code).to.be.eql(HttpStatusCode.UNAUTHORIZED); }); - - async function expectKeys(keymanagerClient: Api, expectedPubkeys: string[], message: string): Promise { - const remoteKeys = await keymanagerClient.listRemoteKeys(); - ApiError.assert(remoteKeys); - expectDeepEquals( - remoteKeys.response.data, - expectedPubkeys.map((pubkey) => ({pubkey, url, readonly: false})), - message - ); - } }); diff --git a/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts b/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts index 12118f1686c8..a57bf87ae016 100644 --- a/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts +++ b/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts @@ -2,14 +2,16 @@ import path from "node:path"; import {rimraf} from "rimraf"; import {Interchange} from "@lodestar/validator"; import {ApiError} from "@lodestar/api"; +import {getMochaContext} from "@lodestar/test-utils/mocha"; import {testFilesDir} from "../utils.js"; -import {describeCliTest} from "../utils/childprocRunner.js"; import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js"; -import {expectDeepEquals, getAfterEachCallbacks} from "../utils/runUtils.js"; -import {getKeymanagerTestRunner} from "../utils/keymanagerTestRunners.js"; +import {expectDeepEquals} from "../utils/runUtils.js"; +import {startValidatorWithKeyManager} from "../utils/validator.js"; import {getKeystoresStr} from "../utils/keystores.js"; -describeCliTest("import keystores from api, test DefaultProposerConfig", function ({spawnCli}) { +describe("import keystores from api, test DefaultProposerConfig", function () { + this.timeout("30s"); + const testContext = getMochaContext(this); const dataDir = path.join(testFilesDir, "proposer-config-test"); const defaultOptions = { @@ -26,9 +28,6 @@ describeCliTest("import keystores from api, test DefaultProposerConfig", functio rimraf.sync(dataDir); }); - const afterEachCallbacks = getAfterEachCallbacks(); - const itKeymanagerStep = getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, dataDir}); - /** Generated from const sk = bls.SecretKey.fromKeygen(Buffer.alloc(32, 0xaa)); */ const passphrase = "AAAAAAAA0000000000"; const keyCount = 2; @@ -47,119 +46,115 @@ describeCliTest("import keystores from api, test DefaultProposerConfig", functio }; const slashingProtectionStr = JSON.stringify(slashingProtection); - itKeymanagerStep( - "1 . run 'validator' import keys from API, getdefaultfeeRecipient", - async function (keymanagerClient) { - // Produce and encrypt keystores - // Import test keys - const keystoresStr = await getKeystoresStr(passphrase, secretKeys); - await keymanagerClient.importKeystores(keystoresStr, passphrases, slashingProtectionStr); - - //////////////// Fee Recipient - - let feeRecipient0 = await keymanagerClient.listFeeRecipient(pubkeys[0]); - ApiError.assert(feeRecipient0); - expectDeepEquals( - feeRecipient0.response.data, - {pubkey: pubkeys[0], ethaddress: defaultOptions.suggestedFeeRecipient}, - "FeeRecipient Check default" - ); - - // Set feeClient to updatedOptions - ApiError.assert(await keymanagerClient.setFeeRecipient(pubkeys[0], updatedOptions.suggestedFeeRecipient)); - feeRecipient0 = await keymanagerClient.listFeeRecipient(pubkeys[0]); - ApiError.assert(feeRecipient0); - expectDeepEquals( - feeRecipient0.response.data, - {pubkey: pubkeys[0], ethaddress: updatedOptions.suggestedFeeRecipient}, - "FeeRecipient Check updated" - ); - - /////////// GasLimit - - let gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); - ApiError.assert(gasLimit0); - expectDeepEquals( - gasLimit0.response.data, - {pubkey: pubkeys[0], gasLimit: defaultOptions.gasLimit}, - "gasLimit Check default" - ); - - // Set GasLimit to updatedOptions - ApiError.assert(await keymanagerClient.setGasLimit(pubkeys[0], updatedOptions.gasLimit)); - gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); - ApiError.assert(gasLimit0); - expectDeepEquals( - gasLimit0.response.data, - {pubkey: pubkeys[0], gasLimit: updatedOptions.gasLimit}, - "gasLimit Check updated" - ); - } - ); - - itKeymanagerStep( - "2 . run 'validator' Check last feeRecipient and gasLimit persists", - async function (keymanagerClient) { - // next time check edited feeRecipient persists - let feeRecipient0 = await keymanagerClient.listFeeRecipient(pubkeys[0]); - ApiError.assert(feeRecipient0); - expectDeepEquals( - feeRecipient0.response.data, - {pubkey: pubkeys[0], ethaddress: updatedOptions.suggestedFeeRecipient}, - "FeeRecipient Check default persists" - ); - - // after deletion feeRecipient restored to default - ApiError.assert(await keymanagerClient.deleteFeeRecipient(pubkeys[0])); - feeRecipient0 = await keymanagerClient.listFeeRecipient(pubkeys[0]); - ApiError.assert(feeRecipient0); - expectDeepEquals( - feeRecipient0.response.data, - {pubkey: pubkeys[0], ethaddress: defaultOptions.suggestedFeeRecipient}, - "FeeRecipient Check default after delete" - ); - - // gasLimit persists - let gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); - ApiError.assert(gasLimit0); - expectDeepEquals( - gasLimit0.response.data, - {pubkey: pubkeys[0], gasLimit: updatedOptions.gasLimit}, - "gasLimit Check updated persists" - ); - - ApiError.assert(await keymanagerClient.deleteGasLimit(pubkeys[0])); - gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); - ApiError.assert(gasLimit0); - expectDeepEquals( - gasLimit0.response.data, - {pubkey: pubkeys[0], gasLimit: defaultOptions.gasLimit}, - "gasLimit Check default after delete" - ); - } - ); - - itKeymanagerStep( - "3 . run 'validator' FeeRecipient and GasLimit should be default after delete", - async function (keymanagerClient) { - const feeRecipient0 = await keymanagerClient.listFeeRecipient(pubkeys[0]); - ApiError.assert(feeRecipient0); - expectDeepEquals( - feeRecipient0.response.data, - {pubkey: pubkeys[0], ethaddress: defaultOptions.suggestedFeeRecipient}, - "FeeRecipient Check default persists" - ); - - let gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); - - ApiError.assert(await keymanagerClient.deleteGasLimit(pubkeys[0])); - gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); - ApiError.assert(gasLimit0); - expectDeepEquals( - gasLimit0.response.data, - {pubkey: pubkeys[0], gasLimit: defaultOptions.gasLimit}, - "gasLimit Check default after delete" - ); - } - ); + it("1 . run 'validator' import keys from API, getdefaultfeeRecipient", async () => { + const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext}); + // Produce and encrypt keystores + // Import test keys + const keystoresStr = await getKeystoresStr(passphrase, secretKeys); + await keymanagerClient.importKeystores(keystoresStr, passphrases, slashingProtectionStr); + + //////////////// Fee Recipient + + let feeRecipient0 = await keymanagerClient.listFeeRecipient(pubkeys[0]); + ApiError.assert(feeRecipient0); + expectDeepEquals( + feeRecipient0.response.data, + {pubkey: pubkeys[0], ethaddress: defaultOptions.suggestedFeeRecipient}, + "FeeRecipient Check default" + ); + + // Set feeClient to updatedOptions + ApiError.assert(await keymanagerClient.setFeeRecipient(pubkeys[0], updatedOptions.suggestedFeeRecipient)); + feeRecipient0 = await keymanagerClient.listFeeRecipient(pubkeys[0]); + ApiError.assert(feeRecipient0); + expectDeepEquals( + feeRecipient0.response.data, + {pubkey: pubkeys[0], ethaddress: updatedOptions.suggestedFeeRecipient}, + "FeeRecipient Check updated" + ); + + /////////// GasLimit + + let gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); + ApiError.assert(gasLimit0); + expectDeepEquals( + gasLimit0.response.data, + {pubkey: pubkeys[0], gasLimit: defaultOptions.gasLimit}, + "gasLimit Check default" + ); + + // Set GasLimit to updatedOptions + ApiError.assert(await keymanagerClient.setGasLimit(pubkeys[0], updatedOptions.gasLimit)); + gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); + ApiError.assert(gasLimit0); + expectDeepEquals( + gasLimit0.response.data, + {pubkey: pubkeys[0], gasLimit: updatedOptions.gasLimit}, + "gasLimit Check updated" + ); + }); + + it("2 . run 'validator' Check last feeRecipient and gasLimit persists", async () => { + const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext}); + + // next time check edited feeRecipient persists + let feeRecipient0 = await keymanagerClient.listFeeRecipient(pubkeys[0]); + ApiError.assert(feeRecipient0); + expectDeepEquals( + feeRecipient0.response.data, + {pubkey: pubkeys[0], ethaddress: updatedOptions.suggestedFeeRecipient}, + "FeeRecipient Check default persists" + ); + + // after deletion feeRecipient restored to default + ApiError.assert(await keymanagerClient.deleteFeeRecipient(pubkeys[0])); + feeRecipient0 = await keymanagerClient.listFeeRecipient(pubkeys[0]); + ApiError.assert(feeRecipient0); + expectDeepEquals( + feeRecipient0.response.data, + {pubkey: pubkeys[0], ethaddress: defaultOptions.suggestedFeeRecipient}, + "FeeRecipient Check default after delete" + ); + + // gasLimit persists + let gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); + ApiError.assert(gasLimit0); + expectDeepEquals( + gasLimit0.response.data, + {pubkey: pubkeys[0], gasLimit: updatedOptions.gasLimit}, + "gasLimit Check updated persists" + ); + + ApiError.assert(await keymanagerClient.deleteGasLimit(pubkeys[0])); + gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); + ApiError.assert(gasLimit0); + expectDeepEquals( + gasLimit0.response.data, + {pubkey: pubkeys[0], gasLimit: defaultOptions.gasLimit}, + "gasLimit Check default after delete" + ); + }); + + it("3 . run 'validator' FeeRecipient and GasLimit should be default after delete", async () => { + const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext}); + + const feeRecipient0 = await keymanagerClient.listFeeRecipient(pubkeys[0]); + ApiError.assert(feeRecipient0); + expectDeepEquals( + feeRecipient0.response.data, + {pubkey: pubkeys[0], ethaddress: defaultOptions.suggestedFeeRecipient}, + "FeeRecipient Check default persists" + ); + + let gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); + + ApiError.assert(await keymanagerClient.deleteGasLimit(pubkeys[0])); + gasLimit0 = await keymanagerClient.getGasLimit(pubkeys[0]); + ApiError.assert(gasLimit0); + expectDeepEquals( + gasLimit0.response.data, + {pubkey: pubkeys[0], gasLimit: defaultOptions.gasLimit}, + "gasLimit Check default after delete" + ); + }); }); diff --git a/packages/cli/test/e2e/runDevCmd.test.ts b/packages/cli/test/e2e/runDevCmd.test.ts index e1acd7ced617..69c8989f1788 100644 --- a/packages/cli/test/e2e/runDevCmd.test.ts +++ b/packages/cli/test/e2e/runDevCmd.test.ts @@ -1,32 +1,41 @@ import {ApiError, getClient} from "@lodestar/api"; import {config} from "@lodestar/config/default"; import {retry} from "@lodestar/utils"; -import {describeCliTest} from "../utils/childprocRunner.js"; -import {itDone} from "../utils/runUtils.js"; +import {spawnCliCommand} from "@lodestar/test-utils"; +import {getMochaContext} from "@lodestar/test-utils/mocha"; -describeCliTest("Run dev command", function ({spawnCli}) { - itDone("Run dev command with no --dataDir until beacon api is listening", async function (done) { +describe("Run dev command", function () { + const testContext = getMochaContext(this); + this.timeout("30s"); + + it("Run dev command with no --dataDir until beacon api is listening", async () => { const beaconPort = 39011; - const devProc = spawnCli({pipeStdToParent: false, printOnlyOnError: true, logPrefix: "dev"}, [ - // ⏎ - "dev", - "--reset", - "--startValidators=0..7", - `--rest.port=${beaconPort}`, - ]); + const devProc = await spawnCliCommand( + "packages/cli/bin/lodestar.js", + ["dev", "--reset", "--startValidators=0..7", `--rest.port=${beaconPort}`], + {pipeStdioToParent: true, logPrefix: "dev", testContext} + ); // Exit early if process exits devProc.on("exit", (code) => { if (code !== null && code > 0) { - done(Error(`process exited with code ${code}`)); + throw new Error(`process exited with code ${code}`); } }); const beaconUrl = `http://127.0.0.1:${beaconPort}`; - const client = getClient({baseUrl: beaconUrl}, {config}); + // To cleanup the event stream connection + const httpClientController = new AbortController(); + const client = getClient({baseUrl: beaconUrl, getAbortSignal: () => httpClientController.signal}, {config}); // Wrap in retry since the API may not be listening yet await retry(() => client.node.getHealth().then((res) => ApiError.assert(res)), {retryDelay: 1000, retries: 60}); + httpClientController.abort(); + + // The process will exit when the test finishes + // Default behavior would be the abort signal will be passed to the child process + // The earlier registered callback will consider it as an error and throw + devProc.removeAllListeners("exit"); }); }); diff --git a/packages/cli/test/e2e/validatorList.test.ts b/packages/cli/test/e2e/validatorList.test.ts index ce2d10e7b56c..ba2102f07fee 100644 --- a/packages/cli/test/e2e/validatorList.test.ts +++ b/packages/cli/test/e2e/validatorList.test.ts @@ -3,26 +3,19 @@ import fs from "node:fs"; import path from "node:path"; import {rimraf} from "rimraf"; import {expect} from "chai"; -import sinon from "sinon"; import {Keystore} from "@chainsafe/bls-keystore"; import {fromHex} from "@lodestar/utils"; +import {runCliCommand} from "@lodestar/test-utils"; +import {stubLogger} from "@lodestar/test-utils/sinon"; import {testFilesDir} from "../utils.js"; -import {getCliInMemoryRunner} from "../utils/inMemoryRunner.js"; +import {getLodestarCli} from "../../src/cli.js"; describe("cmds / validator", function () { - const lodestar = getCliInMemoryRunner(); - + this.timeout("30s"); + stubLogger(this, console); + const lodestar = getLodestarCli(); const dataDir = testFilesDir; - beforeEach(() => { - sinon.spy(console, "info"); - sinon.spy(console, "log"); - }); - - afterEach(() => { - sinon.restore(); - }); - before("Clean dataDir", () => { rimraf.sync(dataDir); }); @@ -41,7 +34,7 @@ describe("cmds / validator", function () { fs.writeFileSync(passphraseFilepath, passphrase); fs.writeFileSync(keystoreFilepath, keystore.stringify()); - await lodestar([ + await runCliCommand(lodestar, [ "validator import", `--dataDir ${dataDir}`, `--keystore ${keystoreFilepath}`, @@ -55,7 +48,7 @@ describe("cmds / validator", function () { fs.mkdirSync(path.join(dataDir, "keystores"), {recursive: true}); fs.mkdirSync(path.join(dataDir, "secrets"), {recursive: true}); - await lodestar(["validator list", `--dataDir ${dataDir}`]); + await runCliCommand(lodestar, ["validator list", `--dataDir ${dataDir}`], {timeoutMs: 5000}); expect(console.info).calledWith("1 local keystores"); expect(console.info).calledWith(pkHex); diff --git a/packages/cli/test/e2e/voluntaryExit.test.ts b/packages/cli/test/e2e/voluntaryExit.test.ts index aa78f143b1d1..b3a539473581 100644 --- a/packages/cli/test/e2e/voluntaryExit.test.ts +++ b/packages/cli/test/e2e/voluntaryExit.test.ts @@ -1,40 +1,48 @@ import path from "node:path"; -import {sleep, retry} from "@lodestar/utils"; +import {retry} from "@lodestar/utils"; import {ApiError, getClient} from "@lodestar/api"; import {config} from "@lodestar/config/default"; import {interopSecretKey} from "@lodestar/state-transition"; +import {spawnCliCommand, execCliCommand} from "@lodestar/test-utils"; +import {getMochaContext} from "@lodestar/test-utils/mocha"; import {testFilesDir} from "../utils.js"; -import {describeCliTest, execCli} from "../utils/childprocRunner.js"; -import {itDone} from "../utils/runUtils.js"; -describeCliTest("voluntaryExit cmd", function ({spawnCli}) { +describe("voluntaryExit cmd", function () { + const testContext = getMochaContext(this); this.timeout("60s"); - itDone("Perform a voluntary exit", async function (done) { + it("Perform a voluntary exit", async () => { const restPort = 9596; - const devBnProc = spawnCli({pipeStdToParent: false, logPrefix: "dev"}, [ - // ⏎ - "dev", - `--dataDir=${path.join(testFilesDir, "dev-voluntary-exit")}`, - "--genesisValidators=8", - "--startValidators=0..7", - "--rest", - `--rest.port=${restPort}`, - // Speed up test to make genesis happen faster - "--params.SECONDS_PER_SLOT=2", - // Allow voluntary exists to be valid immediately - "--params.SHARD_COMMITTEE_PERIOD=0", - ]); + const devBnProc = await spawnCliCommand( + "packages/cli/bin/lodestar.js", + [ + // ⏎ + "dev", + `--dataDir=${path.join(testFilesDir, "dev-voluntary-exit")}`, + "--genesisValidators=8", + "--startValidators=0..7", + "--rest", + `--rest.port=${restPort}`, + // Speed up test to make genesis happen faster + "--params.SECONDS_PER_SLOT=2", + // Allow voluntary exists to be valid immediately + "--params.SHARD_COMMITTEE_PERIOD=0", + ], + {pipeStdioToParent: false, logPrefix: "dev", testContext} + ); + // Exit early if process exits devBnProc.on("exit", (code) => { if (code !== null && code > 0) { - done(Error(`devBnProc process exited with code ${code}`)); + throw new Error(`devBnProc process exited with code ${code}`); } }); const baseUrl = `http://127.0.0.1:${restPort}`; - const client = getClient({baseUrl}, {config}); + // To cleanup the event stream connection + const httpClientController = new AbortController(); + const client = getClient({baseUrl, getAbortSignal: () => httpClientController.signal}, {config}); // Wait for beacon node API to be available + genesis await retry( @@ -55,16 +63,19 @@ describeCliTest("voluntaryExit cmd", function ({spawnCli}) { // 2 0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4 // 3 0x88c141df77cd9d8d7a71a75c826c41a9c9f03c6ee1b180f3e7852f6a280099ded351b58d66e653 - await execCli([ - // ⏎ - "validator", - "voluntary-exit", - "--network=dev", - "--yes", - "--interopIndexes=0..3", - `--server=${baseUrl}`, - `--pubkeys=${pubkeysToExit.join(",")}`, - ]); + await execCliCommand( + "packages/cli/bin/lodestar.js", + [ + "validator", + "voluntary-exit", + "--network=dev", + "--yes", + "--interopIndexes=0..3", + `--server=${baseUrl}`, + `--pubkeys=${pubkeysToExit.join(",")}`, + ], + {pipeStdioToParent: false, logPrefix: "voluntary-exit"} + ); for (const pubkey of pubkeysToExit) { await retry( @@ -82,8 +93,7 @@ describeCliTest("voluntaryExit cmd", function ({spawnCli}) { ); } - devBnProc.kill("SIGINT"); - await sleep(1000); - devBnProc.kill("SIGKILL"); + // Disconnect the event stream for the client + httpClientController.abort(); }); }); diff --git a/packages/cli/test/utils/childprocRunner.ts b/packages/cli/test/utils/childprocRunner.ts deleted file mode 100644 index 5e3a79e5e7a4..000000000000 --- a/packages/cli/test/utils/childprocRunner.ts +++ /dev/null @@ -1,146 +0,0 @@ -import child_process from "node:child_process"; -import {shell, ShellOpts} from "./shell.js"; - -const {RUN_FROM_SRC} = process.env; - -const nodeJsBinaryPath = process.execPath; -const tsNodeBinaryPath = esmRelativePathJoin("../../../../node_modules/.bin/ts-node"); -const cliSrcScriptPath = esmRelativePathJoin("../../src/index.ts"); -const cliLibScriptPath = esmRelativePathJoin("../../lib/index.js"); - -/* eslint-disable no-console */ - -export type DescribeArgs = { - spawnCli(opts: SpawnCliOpts, args: string[]): child_process.ChildProcessWithoutNullStreams; -}; - -type SpawnCliOpts = { - ensureProcRunning?: boolean; - logPrefix?: string; - pipeStdToParent?: boolean; - printOnlyOnError?: boolean; -}; - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function describeCliTest(testName: string, callback: (this: Mocha.Suite, args: DescribeArgs) => void) { - const afterEachCallbacks: (() => Promise | void)[] = []; - afterEach(async () => { - const errs: Error[] = []; - for (const cb of afterEachCallbacks) { - try { - await cb(); - } catch (e) { - errs.push(e as Error); - } - } - afterEachCallbacks.length = 0; // Reset array - if (errs.length > 0) throw errs[0]; - }); - - const args: DescribeArgs = { - spawnCli(opts: SpawnCliOpts, args: string[]) { - const proc = spawnCli(opts, args); - console.log(`Created process ${proc.pid}`); - - afterEachCallbacks.push(async function () { - // Capture state before killing - const killed = proc.killed; - - // Attempt to kill process both with linux tools and built-in .kill() - // Note: `kill ` does not suffice in a local Ubuntu environment. - console.log("Killing process", proc.pid); - proc.kill("SIGKILL"); - await shell(`pkill -P ${proc.pid}`).catch((e) => { - // Do not log unless on debug mode, process is probably killed already - if (process.env.DEBUG) console.error(e); - }); - - if (killed && opts?.ensureProcRunning) { - throw Error(`Process ${proc.pid} already killed`); - } - }); - - return proc; - }, - }; - - describe(testName, function () { - // Extend timeout to allow compiling from src - // TODO: Just build from src once in before - this.timeout(RUN_FROM_SRC ? "60s" : "10s"); - - callback.bind(this)(args); - }); -} - -export function spawnCli(opts: SpawnCliOpts, lodestarArgs: string[]): child_process.ChildProcessWithoutNullStreams { - let stdstr = ""; - const logPrefix = opts?.logPrefix ?? ""; - - const command = RUN_FROM_SRC - ? // ts-node --esm cli.ts - tsNodeBinaryPath - : // node cli.js - nodeJsBinaryPath; - const prefixArgs = RUN_FROM_SRC - ? // ts-node --esm cli.ts - ["--esm", cliSrcScriptPath, ...lodestarArgs] - : // node cli.js - [cliLibScriptPath, ...lodestarArgs]; - - const proc = child_process.spawn(command, prefixArgs); - - if (opts?.pipeStdToParent) { - proc.stdout.on("data", (chunk) => { - const str = Buffer.from(chunk).toString("utf8"); - process.stdout.write(`${logPrefix} ${proc.pid}: ${str}`); // str already contains a new line. console.log adds a new line - }); - proc.stderr.on("data", (chunk) => { - const str = Buffer.from(chunk).toString("utf8"); - process.stderr.write(`${logPrefix} ${proc.pid}: ${str}`); // str already contains a new line. console.log adds a new line - }); - } else { - proc.stdout.on("data", (chunk) => { - stdstr += Buffer.from(chunk).toString("utf8"); - }); - proc.stderr.on("data", (chunk) => { - stdstr += Buffer.from(chunk).toString("utf8"); - }); - } - - proc.on("exit", (code) => { - console.log("process exited", {code}); - if (!opts?.pipeStdToParent) { - if (!opts?.printOnlyOnError || (code !== null && code > 0)) { - console.log(stdstr); - } - } - }); - - return proc; -} - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function bufferStderr(proc: child_process.ChildProcessWithoutNullStreams) { - let data = ""; - proc.stderr.on("data", (chunk) => { - data += Buffer.from(chunk).toString("utf8"); - }); - return { - read: () => data, - }; -} - -export function execCli(lodestarArgs: string[], opts?: ShellOpts): Promise { - const prefixArgs = RUN_FROM_SRC - ? // ts-node --esm cli.ts - [tsNodeBinaryPath, "--esm", cliSrcScriptPath] - : // node cli.js - [nodeJsBinaryPath, cliLibScriptPath]; - return shell([...prefixArgs, ...lodestarArgs], {pipeToProcess: true, ...opts}); -} - -// From https://blog.logrocket.com/alternatives-dirname-node-js-es-modules -function esmRelativePathJoin(relativePath: string): string { - return new URL(relativePath, import.meta.url).toString().replace(/^file:\/\//, ""); -} diff --git a/packages/cli/test/utils/inMemoryRunner.ts b/packages/cli/test/utils/inMemoryRunner.ts deleted file mode 100644 index 72f4ef2b66fa..000000000000 --- a/packages/cli/test/utils/inMemoryRunner.ts +++ /dev/null @@ -1,26 +0,0 @@ -import yargs from "yargs"; -import {getLodestarCli} from "../../src/cli.js"; - -export function getCliInMemoryRunner() { - return async (arg: string | readonly string[], context?: Record): Promise => { - return new Promise((resolve, reject) => { - const lodestar = getLodestarCli() as yargs.Argv; - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call - lodestar - // Method to execute when a failure occurs, rather than printing the failure message. - .fail((msg, err) => { - if (err !== undefined) reject(err); - else if (msg) reject(Error(msg)); - else reject(Error("Unknown error")); - }) - .help(false) - .exitProcess(false) - .parse(Array.isArray(arg) ? arg.join(" ") : arg, context) - // Called after the completion of any command. handler is invoked with the result returned by the command: - .then((result: any) => { - resolve(result); - }) - .catch((e: unknown) => reject(e)); - }); - }; -} diff --git a/packages/cli/test/utils/keymanagerTestRunners.ts b/packages/cli/test/utils/keymanagerTestRunners.ts deleted file mode 100644 index ce470c93ac04..000000000000 --- a/packages/cli/test/utils/keymanagerTestRunners.ts +++ /dev/null @@ -1,87 +0,0 @@ -import {sleep, retry} from "@lodestar/utils"; -import {Api, getClient} from "@lodestar/api/keymanager"; -import {config} from "@lodestar/config/default"; -import {ApiError} from "@lodestar/api"; -import {getMockBeaconApiServer} from "./mockBeaconApiServer.js"; -import {AfterEachCallback, expectDeepEqualsUnordered, findApiToken, itDone} from "./runUtils.js"; -import {DescribeArgs} from "./childprocRunner.js"; - -type TestContext = { - args: DescribeArgs; - afterEachCallbacks: AfterEachCallback[]; - dataDir: string; -}; - -type KeymanagerStepOpts = { - validatorCmdExtraArgs?: string[]; -}; - -type KeymanagerStepCbArgs = { - keymanagerUrl: string; -}; - -export function getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, dataDir}: TestContext) { - return function itKeymanagerStep( - itName: string, - cb: (this: Mocha.Context, keymanagerClient: Api, args: KeymanagerStepCbArgs) => Promise, - keymanagerStepOpts?: KeymanagerStepOpts - ): void { - itDone(itName, async function (done) { - this.timeout("60s"); - - const keymanagerPort = 38011; - const beaconPort = 39011; - const keymanagerUrl = `http://localhost:${keymanagerPort}`; - const beaconUrl = `http://localhost:${beaconPort}`; - - const beaconServer = getMockBeaconApiServer({port: beaconPort}); - afterEachCallbacks.push(() => beaconServer.close()); - await beaconServer.listen(); - - const validatorProc = spawnCli({pipeStdToParent: true, logPrefix: "vc"}, [ - // ⏎ - "validator", - `--dataDir=${dataDir}`, - "--keymanager", - "--keymanager.address=localhost", - `--keymanager.port=${keymanagerPort}`, - `--server=${beaconUrl}`, - ...(keymanagerStepOpts?.validatorCmdExtraArgs ?? []), - ]); - // Exit early if process exits - validatorProc.on("exit", (code) => { - if (code !== null && code > 0) { - done(Error(`process exited with code ${code}`)); - } - }); - - // Wait for api-token.txt file to be written to disk and find it - const apiToken = await retry(async () => findApiToken(dataDir), {retryDelay: 500, retries: 10}); - - const keymanagerClient = getClient({baseUrl: keymanagerUrl, bearerToken: apiToken}, {config}); - - // Wrap in retry since the API may not be listening yet - await retry(() => keymanagerClient.listRemoteKeys(), {retryDelay: 500, retries: 10}); - - await cb.bind(this)(keymanagerClient, {keymanagerUrl}); - - validatorProc.kill("SIGINT"); - await sleep(1000); - validatorProc.kill("SIGKILL"); - }); - }; -} - -/** - * Query `keymanagerClient.listKeys()` API endpoint and assert that expectedPubkeys are in the response - */ -export async function expectKeys(keymanagerClient: Api, expectedPubkeys: string[], message: string): Promise { - const keys = await keymanagerClient.listKeys(); - ApiError.assert(keys); - // The order of keys isn't always deterministic so we can't use deep equal - expectDeepEqualsUnordered( - keys.response.data, - expectedPubkeys.map((pubkey) => ({validatingPubkey: pubkey, derivationPath: "", readonly: false})), - message - ); -} diff --git a/packages/cli/test/utils/runUtils.ts b/packages/cli/test/utils/runUtils.ts index 7050460be2bb..f6a9c311946b 100644 --- a/packages/cli/test/utils/runUtils.ts +++ b/packages/cli/test/utils/runUtils.ts @@ -26,45 +26,3 @@ export function expectDeepEquals(a: T, b: T, message: string): void { export function expectDeepEqualsUnordered(a: T[], b: T[], message: string): void { expect(a).to.have.deep.members(b, message); } - -export type DoneCb = (err?: Error) => void; - -/** - * Extends Mocha it() to allow BOTH: - * - Resolve / reject callback promise to end test - * - Use done() to end test early - */ -export function itDone(itName: string, cb: (this: Mocha.Context, done: DoneCb) => Promise): void { - it(itName, function () { - return new Promise((resolve, reject) => { - function done(err?: Error): void { - if (err) reject(err); - else resolve(); - } - cb.bind(this)(done).then(resolve, reject); - }); - }); -} - -export type AfterEachCallback = () => Promise | void; - -export function getAfterEachCallbacks(): AfterEachCallback[] { - const afterEachCallbacks: (() => Promise | void)[] = []; - - afterEach(async () => { - const errs: Error[] = []; - for (const cb of afterEachCallbacks) { - try { - await cb(); - } catch (e) { - errs.push(e as Error); - } - } - afterEachCallbacks.length = 0; // Reset array - if (errs.length > 0) { - throw errs[0]; - } - }); - - return afterEachCallbacks; -} diff --git a/packages/cli/test/utils/shell.ts b/packages/cli/test/utils/shell.ts deleted file mode 100644 index 4cd4d9473232..000000000000 --- a/packages/cli/test/utils/shell.ts +++ /dev/null @@ -1,50 +0,0 @@ -import childProcess from "node:child_process"; - -/** - * If timeout is greater than 0, the parent will send the signal - * identified by the killSignal property (the default is 'SIGTERM') - * if the child runs longer than timeout milliseconds. - */ -const defaultTimeout = 15 * 60 * 1000; // ms - -export type ShellOpts = { - timeout?: number; - maxBuffer?: number; - signal?: AbortSignal; - pipeToProcess?: boolean; -}; - -/** - * Run arbitrary commands in a shell - * If the child process exits with code > 0, rejects - */ -export async function shell(cmd: string | string[], options?: ShellOpts): Promise { - const timeout = options?.timeout ?? defaultTimeout; - const maxBuffer = options?.maxBuffer; - const cmdStr = Array.isArray(cmd) ? cmd.join(" ") : cmd; - - return new Promise((resolve, reject) => { - const proc = childProcess.exec(cmdStr, {timeout, maxBuffer}, (err, stdout) => { - if (err) { - reject(err); - } else { - resolve(stdout.trim()); - } - }); - - if (options?.pipeToProcess) { - proc.stdout?.pipe(process.stdout); - proc.stderr?.pipe(process.stderr); - } - - if (options?.signal) { - options.signal.addEventListener( - "abort", - () => { - proc.kill("SIGKILL"); - }, - {once: true} - ); - } - }); -} diff --git a/packages/cli/test/utils/simulation/runner/ChildProcessRunner.ts b/packages/cli/test/utils/simulation/runner/ChildProcessRunner.ts index a736359a361d..8554a2d9a3be 100644 --- a/packages/cli/test/utils/simulation/runner/ChildProcessRunner.ts +++ b/packages/cli/test/utils/simulation/runner/ChildProcessRunner.ts @@ -1,6 +1,12 @@ import {ChildProcess} from "node:child_process"; +import { + spawnChildProcess, + stopChildProcess, + ChildProcessHealthStatus, + SpawnChildProcessOptions, + ChildProcessResolve, +} from "@lodestar/test-utils"; import {Job, JobOptions, RunnerEnv, RunnerType} from "../interfaces.js"; -import {startChildProcess, stopChildProcess} from "../utils/child_process.js"; export class ChildProcessRunner implements RunnerEnv { type = RunnerType.ChildProcess as const; @@ -8,10 +14,32 @@ export class ChildProcessRunner implements RunnerEnv { create(jobOption: Omit, "children">): Job { let childProcess: ChildProcess; + const spawnOpts: SpawnChildProcessOptions = { + env: jobOption.cli.env, + pipeStdioToFile: jobOption.logs.stdoutFilePath, + logPrefix: jobOption.id, + }; + + const health = jobOption.health; + + if (health) { + spawnOpts.healthTimeoutMs = 30000; + spawnOpts.health = async (): Promise => + health() + .then((status) => { + return status.ok ? {healthy: true} : {healthy: false}; + }) + .catch((error) => { + return {healthy: false, message: (error as Error).message}; + }); + } else { + spawnOpts.resolveOn = ChildProcessResolve.Completion; + } + return { id: jobOption.id, start: async () => { - childProcess = await startChildProcess(jobOption); + childProcess = await spawnChildProcess(jobOption.cli.command, jobOption.cli.args, spawnOpts); }, stop: async () => { if (childProcess === undefined) { diff --git a/packages/cli/test/utils/simulation/runner/DockerRunner.ts b/packages/cli/test/utils/simulation/runner/DockerRunner.ts index 0e809d28f159..91dcb492c3eb 100644 --- a/packages/cli/test/utils/simulation/runner/DockerRunner.ts +++ b/packages/cli/test/utils/simulation/runner/DockerRunner.ts @@ -1,8 +1,15 @@ /* eslint-disable no-console */ import {ChildProcess} from "node:child_process"; import {sleep} from "@lodestar/utils"; +import { + ChildProcessHealthStatus, + SpawnChildProcessOptions, + execChildProcess, + spawnChildProcess, + stopChildProcess, + ChildProcessResolve, +} from "@lodestar/test-utils"; import {Job, JobOptions, RunnerEnv, RunnerType} from "../interfaces.js"; -import {startChildProcess, stopChildProcess} from "../utils/child_process.js"; const dockerNetworkIpRange = "192.168.0"; const dockerNetworkName = "sim-env-net"; @@ -19,15 +26,9 @@ export class DockerRunner implements RunnerEnv { async start(): Promise { try { - await startChildProcess({ - id: `create docker network '${dockerNetworkName}'`, - cli: { - command: "docker", - args: ["network", "create", "--subnet", `${dockerNetworkIpRange}.0/24`, dockerNetworkName], - }, - logs: { - stdoutFilePath: this.logFilePath, - }, + await execChildProcess(`docker network create --subnet ${dockerNetworkIpRange}.0/24 ${dockerNetworkName}`, { + logPrefix: "docker-runner", + pipeStdioToFile: this.logFilePath, }); } catch { // During multiple sim tests files the network might already exist @@ -38,15 +39,9 @@ export class DockerRunner implements RunnerEnv { // Wait for couple of seconds to allow docker to cleanup containers to network connections for (let i = 0; i < 5; i++) { try { - await startChildProcess({ - id: `docker network rm '${dockerNetworkName}'`, - cli: { - command: "docker", - args: ["network", "rm", dockerNetworkName], - }, - logs: { - stdoutFilePath: this.logFilePath, - }, + await execChildProcess(`docker network rm ${dockerNetworkName}`, { + logPrefix: "docker-runner", + pipeStdioToFile: this.logFilePath, }); return; } catch { @@ -94,15 +89,32 @@ export class DockerRunner implements RunnerEnv { let childProcess: ChildProcess; + const spawnOpts: SpawnChildProcessOptions = { + env: jobOption.cli.env, + pipeStdioToFile: jobOption.logs.stdoutFilePath, + logPrefix: jobOption.id, + }; + + const health = jobOption.health; + + if (health) { + spawnOpts.healthTimeoutMs = 30000; + spawnOpts.health = async (): Promise => + health() + .then((status) => { + return status.ok ? {healthy: true} : {healthy: false}; + }) + .catch((error) => { + return {healthy: false, message: (error as Error).message}; + }); + } else { + spawnOpts.resolveOn = ChildProcessResolve.Completion; + } + return { id: jobOption.id, start: async () => { - childProcess = await startChildProcess({ - id: jobOption.id, - logs: jobOption.logs, - cli: {...jobOption.cli, command: "docker", args: jobArgs}, - health: jobOption.health, - }); + childProcess = await spawnChildProcess("docker", jobArgs, spawnOpts); }, stop: async () => { if (childProcess === undefined) { diff --git a/packages/cli/test/utils/simulation/utils/child_process.ts b/packages/cli/test/utils/simulation/utils/child_process.ts deleted file mode 100644 index 094dbc988ed8..000000000000 --- a/packages/cli/test/utils/simulation/utils/child_process.ts +++ /dev/null @@ -1,97 +0,0 @@ -/* eslint-disable no-console */ -import {ChildProcess, spawn} from "node:child_process"; -import fs from "node:fs"; -import path from "node:path"; -import {JobOptions, RunnerType} from "../interfaces.js"; - -const healthCheckIntervalMs = 1000; -const logHealthChecksAfterMs = 2000; - -export const stopChildProcess = async ( - childProcess: ChildProcess, - signal: NodeJS.Signals | number = "SIGTERM" -): Promise => { - if (childProcess.killed || childProcess.exitCode !== null || childProcess.signalCode !== null) { - return; - } - - return new Promise((resolve, reject) => { - childProcess.once("error", reject); - childProcess.once("close", resolve); - childProcess.kill(signal); - }); -}; - -export const startChildProcess = async ( - jobOptions: Pick, "cli" | "logs" | "id" | "health"> -): Promise => { - return new Promise((resolve, reject) => { - void (async () => { - const childProcess = spawn(jobOptions.cli.command, jobOptions.cli.args, { - env: {...process.env, ...jobOptions.cli.env}, - }); - - fs.mkdirSync(path.dirname(jobOptions.logs.stdoutFilePath), {recursive: true}); - const stdoutFileStream = fs.createWriteStream(jobOptions.logs.stdoutFilePath); - childProcess.stdout?.pipe(stdoutFileStream); - childProcess.stderr?.pipe(stdoutFileStream); - - // If there is any error in running the child process, reject the promise - childProcess.on("error", reject); - - // If there is a health check, wait for it to pass - const health = jobOptions.health; - - // If there is a health check, wait for it to pass - if (health) { - const startHealthCheckMs = Date.now(); - const intervalId = setInterval(() => { - health() - .then((isHealthy) => { - if (isHealthy.ok) { - clearInterval(intervalId); - childProcess.removeAllListeners("exit"); - resolve(childProcess); - } else { - const timeSinceHealthCheckStart = Date.now() - startHealthCheckMs; - if (timeSinceHealthCheckStart > logHealthChecksAfterMs) { - console.log(`Health check unsuccessful '${jobOptions.id}' after ${timeSinceHealthCheckStart} ms`); - } - } - }) - .catch((e) => { - console.error("error on health check, health functions must never throw", e); - }); - }, healthCheckIntervalMs); - - childProcess.once("exit", (code: number) => { - clearInterval(intervalId); - stdoutFileStream.close(); - reject( - new Error( - `process exited. job=${jobOptions.id}, code=${code}, command="${ - jobOptions.cli.command - } ${jobOptions.cli.args.join(" ")}"` - ) - ); - }); - } else { - // If there is no health check, resolve/reject on completion - childProcess.once("exit", (code: number) => { - stdoutFileStream.close(); - if (code > 0) { - reject( - new Error( - `process exited. job=${jobOptions.id}, code=${code}, command="${ - jobOptions.cli.command - } ${jobOptions.cli.args.join(" ")}"` - ) - ); - } else { - resolve(childProcess); - } - }); - } - })(); - }); -}; diff --git a/packages/cli/test/utils/validator.ts b/packages/cli/test/utils/validator.ts new file mode 100644 index 000000000000..e6eca55603ed --- /dev/null +++ b/packages/cli/test/utils/validator.ts @@ -0,0 +1,102 @@ +import childProcess from "node:child_process"; +import {retry} from "@lodestar/utils"; +import {Api, getClient} from "@lodestar/api/keymanager"; +import {config} from "@lodestar/config/default"; +import {ApiError} from "@lodestar/api"; +import {spawnCliCommand, gracefullyStopChildProcess} from "@lodestar/test-utils"; +import {TestContext} from "@lodestar/test-utils/mocha"; +import {getMockBeaconApiServer} from "./mockBeaconApiServer.js"; +import {expectDeepEqualsUnordered, findApiToken} from "./runUtils.js"; + +export async function startValidatorWithKeyManager( + args: string[], + { + dataDir, + logPrefix, + testContext, + }: { + dataDir: string; + testContext?: TestContext; + logPrefix?: string; + } +): Promise<{ + validator: childProcess.ChildProcessWithoutNullStreams; + stopValidator: () => Promise; + keymanagerClient: Api; +}> { + const keymanagerPort = 38011; + const beaconPort = 39011; + const keymanagerUrl = `http://localhost:${keymanagerPort}`; + const beaconUrl = `http://localhost:${beaconPort}`; + const beaconServer = getMockBeaconApiServer({port: beaconPort}); + + await beaconServer.listen(); + + const validatorProc = await spawnCliCommand( + "packages/cli/bin/lodestar.js", + [ + "validator", + `--dataDir=${dataDir}`, + "--keymanager", + "--keymanager.address=localhost", + `--keymanager.port=${keymanagerPort}`, + `--server=${beaconUrl}`, + ...(args ?? []), + ], + {pipeStdioToParent: true, logPrefix: logPrefix ?? "vc"} + ); + + // Exit early if process exits + validatorProc.on("exit", (code) => { + if (code !== null && code > 0) { + throw new Error(`process exited with code ${code}`); + } + }); + + // Wait for api-token.txt file to be written to disk and find it + const apiToken = await retry(async () => findApiToken(dataDir), {retryDelay: 500, retries: 10}); + const controller = new AbortController(); + const keymanagerClient = getClient( + {baseUrl: keymanagerUrl, bearerToken: apiToken, getAbortSignal: () => controller.signal}, + {config} + ); + + // Wrap in retry since the API may not be listening yet + // Remote key endpoint takes a while to be ready + await retry(() => keymanagerClient.listRemoteKeys(), {retryDelay: 500, retries: 20}); + + validatorProc.addListener("exit", () => { + controller.abort(); + }); + + const stopValidator = async (): Promise => { + validatorProc.removeAllListeners("exit"); + controller.abort(); + await beaconServer.close(); + await gracefullyStopChildProcess(validatorProc, 3000); + }; + + if (testContext) { + testContext.afterEach(stopValidator); + } + + return { + validator: validatorProc, + stopValidator, + keymanagerClient, + }; +} + +/** + * Query `keymanagerClient.listKeys()` API endpoint and assert that expectedPubkeys are in the response + */ +export async function expectKeys(keymanagerClient: Api, expectedPubkeys: string[], message: string): Promise { + const keys = await keymanagerClient.listKeys(); + ApiError.assert(keys); + // The order of keys isn't always deterministic so we can't use deep equal + expectDeepEqualsUnordered( + keys.response.data, + expectedPubkeys.map((pubkey) => ({validatingPubkey: pubkey, derivationPath: "", readonly: false})), + message + ); +} diff --git a/packages/prover/package.json b/packages/prover/package.json index e060e3abb28d..f4da974ff4af 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -76,13 +76,15 @@ "ethereum-cryptography": "^1.2.0", "find-up": "^6.3.0", "http-proxy": "^1.18.1", + "js-yaml": "^4.1.0", "source-map-support": "^0.5.21", - "winston": "^3.8.2", "winston-transport": "^4.5.0", + "winston": "^3.8.2", "yargs": "^17.7.1" }, "devDependencies": { "@lodestar/logger": "^1.9.1", + "@lodestar/test-utils": "^1.9.1", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/prover/src/cli/applyPreset.ts b/packages/prover/src/cli/applyPreset.ts new file mode 100644 index 000000000000..a6a3568c5f91 --- /dev/null +++ b/packages/prover/src/cli/applyPreset.ts @@ -0,0 +1,81 @@ +// MUST import this file first before anything and not import any Lodestar code. +// +// ## Rationale +// +// Lodestar implemented PRESET / CONFIG separation to allow importing types and preset constants directly +// see https://github.com/ChainSafe/lodestar/pull/2585 +// +// However this prevents dynamic configuration changes which is exactly what the CLI required before. +// - The dev command can't apply the minimal preset dynamically +// - `--network gnosis` can't apply a different preset dynamically +// - `--network chiado` can't apply a different preset dynamically +// +// Running this file allows us to keep a static export strategy while NOT requiring users to +// set LODESTAR_PRESET manually every time. + +// IMPORTANT: only import Lodestar code here which does not import any other Lodestar libraries +import {setActivePreset, presetFromJson, PresetName} from "@lodestar/params/setPreset"; +import {readFile} from "../utils/file.js"; + +const network = valueOfArg("network"); +const preset = valueOfArg("preset"); +const presetFile = valueOfArg("presetFile"); + +// Apply preset flag if present +if (preset) { + process.env.LODESTAR_PRESET = preset; +} + +// If ENV is set overrides, network (otherwise can not override network --dev in mainnet mode) +else if (process.env.LODESTAR_PRESET) { + // break +} + +// Translate network to preset +else if (network) { + if (network === "dev") { + process.env.LODESTAR_PRESET = "minimal"; + // "c-kzg" has hardcoded the mainnet value, do not use presets + // eslint-disable-next-line @typescript-eslint/naming-convention + setActivePreset(PresetName.minimal, {FIELD_ELEMENTS_PER_BLOB: 4096}); + } else if (network === "gnosis" || network === "chiado") { + process.env.LODESTAR_PRESET = "gnosis"; + } +} + +if (presetFile) { + // Override the active preset with custom values from file + // Do not modify the preset to use as a base by passing null + setActivePreset(null, presetFromJson(readFile(presetFile) ?? {})); +} + +/** + * Valid syntax + * - `--preset minimal` + * - `--preset=minimal` + */ +function valueOfArg(argName: string): string | null { + // Syntax `--preset minimal` + // process.argv = ["--preset", "minimal"]; + + { + const index = process.argv.indexOf(`--${argName}`); + if (index > -1) { + return process.argv[index + 1] ?? ""; + } + } + + // Syntax `--preset=minimal` + { + const prefix = `--${argName}=`; + const item = process.argv.find((arg) => arg.startsWith(prefix)); + if (item) { + return item.slice(prefix.length); + } + } + + return null; +} + +// Add empty export to make this a module +export {}; diff --git a/packages/prover/src/cli/cmds/start/handler.ts b/packages/prover/src/cli/cmds/start/handler.ts index c9706602a2d9..5f13db0cfcb6 100644 --- a/packages/prover/src/cli/cmds/start/handler.ts +++ b/packages/prover/src/cli/cmds/start/handler.ts @@ -1,4 +1,6 @@ +import {ChainConfig, chainConfigFromJson} from "@lodestar/config"; import {LCTransport} from "../../../interfaces.js"; +import {readFile} from "../../../utils/file.js"; import {createVerifiedExecutionProxy, VerifiedProxyOptions} from "../../../web3_proxy.js"; import {GlobalArgs, parseGlobalArgs} from "../../options.js"; import {parseStartArgs, StartArgs} from "./options.js"; @@ -7,17 +9,19 @@ import {parseStartArgs, StartArgs} from "./options.js"; * Runs a beacon node. */ export async function proverProxyStartHandler(args: StartArgs & GlobalArgs): Promise { - const {network, logLevel} = parseGlobalArgs(args); + const {network, logLevel, paramsFile} = parseGlobalArgs(args); const opts = parseStartArgs(args); const {executionRpcUrl, port, wsCheckpoint} = opts; + const config: Partial = paramsFile ? chainConfigFromJson(readFile(paramsFile)) : {}; + const options: VerifiedProxyOptions = { logLevel, - network, executionRpcUrl, wsCheckpoint, unverifiedWhitelist: opts.unverifiedWhitelist, requestTimeout: opts.requestTimeout, + ...(network ? {network} : {config}), ...(opts.transport === LCTransport.Rest ? {transport: LCTransport.Rest, urls: opts.urls} : {transport: LCTransport.P2P, bootnodes: opts.bootnodes}), diff --git a/packages/prover/src/cli/index.ts b/packages/prover/src/cli/index.ts index 687c3bb301da..53a32a02eb87 100644 --- a/packages/prover/src/cli/index.ts +++ b/packages/prover/src/cli/index.ts @@ -1,6 +1,7 @@ #!/usr/bin/env node // MUST import first to apply preset from args +import "./applyPreset.js"; import {YargsError} from "../utils/errors.js"; import {getLodestarProverCli, yarg} from "./cli.js"; import "source-map-support/register.js"; diff --git a/packages/prover/src/cli/options.ts b/packages/prover/src/cli/options.ts index 3a5eb4cc49af..cb6ba1aaeca2 100644 --- a/packages/prover/src/cli/options.ts +++ b/packages/prover/src/cli/options.ts @@ -1,22 +1,35 @@ import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; import {LogLevel, LogLevels} from "@lodestar/utils"; +import {ACTIVE_PRESET} from "@lodestar/params"; import {CliCommandOptions} from "../utils/command.js"; export type GlobalArgs = { network: string; logLevel: string; + presetFile?: string; + preset: string; + paramsFile: string; }; export type GlobalOptions = { logLevel: LogLevel; - network: NetworkName; -}; +} & ({paramsFile: string; network?: never} | {network: NetworkName; paramsFile?: never}); export const globalOptions: CliCommandOptions = { network: { description: "Specify the network to connect.", type: "string", - choices: Object.keys(networksChainConfig), + choices: [ + ...Object.keys(networksChainConfig), // Leave always as last network. The order matters for the --help printout + "dev", + ], + conflicts: ["paramsFile"], + }, + + paramsFile: { + description: "Network configuration file", + type: "string", + conflicts: ["network"], }, logLevel: { @@ -25,12 +38,32 @@ export const globalOptions: CliCommandOptions = { choices: LogLevels, default: "info", }, + + // hidden option to allow for LODESTAR_PRESET to be set + preset: { + hidden: true, + type: "string", + default: ACTIVE_PRESET, + }, + + presetFile: { + hidden: true, + description: "Preset configuration file to override the active preset with custom values", + type: "string", + }, }; export function parseGlobalArgs(args: GlobalArgs): GlobalOptions { // Remove undefined values to allow deepmerge to inject default values downstream + if (args.network) { + return { + network: args.network as NetworkName, + logLevel: args.logLevel as LogLevel, + }; + } + return { - network: args.network as NetworkName, logLevel: args.logLevel as LogLevel, + paramsFile: args.paramsFile, }; } diff --git a/packages/prover/src/utils/file.ts b/packages/prover/src/utils/file.ts new file mode 100644 index 000000000000..d236d2d5dc95 --- /dev/null +++ b/packages/prover/src/utils/file.ts @@ -0,0 +1,51 @@ +import fs from "node:fs"; +import path from "node:path"; +import yaml from "js-yaml"; +const {load, FAILSAFE_SCHEMA, Type} = yaml; + +enum FileFormat { + json = "json", + yaml = "yaml", + yml = "yml", + toml = "toml", +} + +const yamlSchema = FAILSAFE_SCHEMA.extend({ + implicit: [ + new Type("tag:yaml.org,2002:str", { + kind: "scalar", + construct: function construct(data) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return data !== null ? data : ""; + }, + }), + ], +}); + +/** + * Parse file contents as Json. + */ +function parse(contents: string, fileFormat: FileFormat): T { + switch (fileFormat) { + case FileFormat.json: + return JSON.parse(contents) as T; + case FileFormat.yaml: + case FileFormat.yml: + return load(contents, {schema: yamlSchema}) as T; + default: + return contents as unknown as T; + } +} + +/** + * Read a JSON serializable object from a file + * + * Parse either from json, yaml, or toml + * Optional acceptedFormats object can be passed which can be an array of accepted formats, in future can be extended to include parseFn for the accepted formats + */ +export function readFile(filepath: string, acceptedFormats?: string[]): T { + const fileFormat = path.extname(filepath).substr(1); + if (acceptedFormats && !acceptedFormats.includes(fileFormat)) throw new Error(`UnsupportedFileFormat: ${filepath}`); + const contents = fs.readFileSync(filepath, "utf-8"); + return parse(contents, fileFormat as FileFormat); +} diff --git a/packages/prover/src/web3_proxy.ts b/packages/prover/src/web3_proxy.ts index c8e7af26519f..508aed45b86c 100644 --- a/packages/prover/src/web3_proxy.ts +++ b/packages/prover/src/web3_proxy.ts @@ -66,7 +66,7 @@ export function createVerifiedExecutionProxy(opts: VerifiedProxyOptions): { } { const {executionRpcUrl, requestTimeout} = opts; const signal = opts.signal ?? new AbortController().signal; - const logger = opts.logger ?? getNodeLogger({level: opts.logLevel ?? LogLevel.info}); + const logger = opts.logger ?? getNodeLogger({level: opts.logLevel ?? LogLevel.info, module: "prover"}); const proofProvider = ProofProvider.init({ ...opts, diff --git a/packages/prover/test/e2e/cli/cmds/start.test.ts b/packages/prover/test/e2e/cli/cmds/start.test.ts new file mode 100644 index 000000000000..941f2c4e71c6 --- /dev/null +++ b/packages/prover/test/e2e/cli/cmds/start.test.ts @@ -0,0 +1,92 @@ +import childProcess from "node:child_process"; +import {writeFile, mkdir} from "node:fs/promises"; +import path from "node:path"; +import {expect} from "chai"; +import Web3 from "web3"; +import {runCliCommand, spawnCliCommand, stopChildProcess} from "@lodestar/test-utils"; +import {sleep} from "@lodestar/utils"; +import {ChainConfig, chainConfigToJson} from "@lodestar/config"; +import {getLodestarProverCli} from "../../../../src/cli/cli.js"; +import {rpcUrl, beaconUrl, proxyPort, proxyUrl, chainId, waitForCapellaFork, config} from "../../../utils/e2e_env.js"; + +const cli = getLodestarProverCli(); + +describe("prover/start", () => { + it("should show help", async () => { + const output = await runCliCommand(cli, ["start", "--help"]); + + expect(output).contains("Show help"); + }); + + it("should fail when --executionRpcUrl is missing", async () => { + await expect(runCliCommand(cli, ["start", "--port", "8088"])).eventually.rejectedWith( + "Missing required argument: executionRpcUrl" + ); + }); + + it("should fail when --beaconUrls and --beaconBootnodes are provided together", async () => { + await expect( + runCliCommand(cli, [ + "start", + "--beaconUrls", + "http://localhost:4000", + "--beaconBootnodes", + "http://localhost:0000", + ]) + ).eventually.rejectedWith("Arguments beaconBootnodes and beaconUrls are mutually exclusive"); + }); + + it("should fail when both of --beaconUrls and --beaconBootnodes are not provided", async () => { + await expect( + runCliCommand(cli, ["start", "--port", "8088", "--executionRpcUrl", "http://localhost:3000"]) + ).eventually.rejectedWith("Either --beaconUrls or --beaconBootnodes must be provided"); + }); + + describe("when started", () => { + let proc: childProcess.ChildProcess; + const paramsFilePath = path.join("/tmp", "e2e-test-env", "params.json"); + const web3: Web3 = new Web3(proxyUrl); + + before(async function () { + this.timeout(50000); + await waitForCapellaFork(); + await mkdir(path.dirname(paramsFilePath), {recursive: true}); + await writeFile(paramsFilePath, JSON.stringify(chainConfigToJson(config as ChainConfig))); + + proc = await spawnCliCommand( + "packages/prover/bin/lodestar-prover.js", + [ + "start", + "--port", + String(proxyPort as number), + "--executionRpcUrl", + rpcUrl, + "--beaconUrls", + beaconUrl, + "--preset", + "minimal", + "--paramsFile", + paramsFilePath, + ], + {runWith: "ts-node", pipeStdioToParent: true} + ); + // Give sometime to the prover to start proxy server + await sleep(3000); + }); + + after(async () => { + await stopChildProcess(proc); + }); + + it("should respond to verified calls", async () => { + const accounts = await web3.eth.getAccounts(); + + expect(accounts.length).to.be.gt(0); + await expect(web3.eth.getBalance(accounts[0])).eventually.not.null; + }); + + it("should respond to unverified calls", async () => { + await expect(web3.eth.getChainId()).eventually.eql(chainId); + }); + }); +}); diff --git a/packages/prover/test/e2e/web3_batch_request.test.ts b/packages/prover/test/e2e/web3_batch_request.test.ts index d44ee8145e4f..472c04274764 100644 --- a/packages/prover/test/e2e/web3_batch_request.test.ts +++ b/packages/prover/test/e2e/web3_batch_request.test.ts @@ -3,7 +3,7 @@ import {expect} from "chai"; import Web3 from "web3"; import {LCTransport} from "../../src/interfaces.js"; import {createVerifiedExecutionProvider} from "../../src/web3_provider.js"; -import {rpcURL, beaconUrl, config} from "../utils/e2e_env.js"; +import {rpcUrl, beaconUrl, config} from "../utils/e2e_env.js"; describe("web3_batch_requests", function () { // Give some margin to sync light client @@ -12,7 +12,7 @@ describe("web3_batch_requests", function () { let web3: Web3; before(() => { - const {provider} = createVerifiedExecutionProvider(new Web3.providers.HttpProvider(rpcURL), { + const {provider} = createVerifiedExecutionProvider(new Web3.providers.HttpProvider(rpcUrl), { transport: LCTransport.Rest, urls: [beaconUrl], config, diff --git a/packages/prover/test/e2e/web3_provider.test.ts b/packages/prover/test/e2e/web3_provider.test.ts index ad2982023ead..b2b4b94277d8 100644 --- a/packages/prover/test/e2e/web3_provider.test.ts +++ b/packages/prover/test/e2e/web3_provider.test.ts @@ -4,7 +4,7 @@ import Web3 from "web3"; import {ethers} from "ethers"; import {LCTransport} from "../../src/interfaces.js"; import {createVerifiedExecutionProvider} from "../../src/web3_provider.js"; -import {waitForCapellaFork, testTimeout, rpcURL, beaconUrl, config} from "../utils/e2e_env.js"; +import {waitForCapellaFork, testTimeout, rpcUrl, beaconUrl, config} from "../utils/e2e_env.js"; describe("web3_provider", function () { this.timeout(testTimeout); @@ -16,7 +16,7 @@ describe("web3_provider", function () { describe("createVerifiedExecutionProvider", () => { describe("web3", () => { it("should connect to the network and call a non-verified method", async () => { - const {provider} = createVerifiedExecutionProvider(new Web3.providers.HttpProvider(rpcURL), { + const {provider} = createVerifiedExecutionProvider(new Web3.providers.HttpProvider(rpcUrl), { transport: LCTransport.Rest, urls: [beaconUrl], config, @@ -33,7 +33,7 @@ describe("web3_provider", function () { describe("ethers", () => { it("should connect to the network and call a non-verified method", async () => { - const {provider} = createVerifiedExecutionProvider(new ethers.JsonRpcProvider(rpcURL), { + const {provider} = createVerifiedExecutionProvider(new ethers.JsonRpcProvider(rpcUrl), { transport: LCTransport.Rest, urls: [beaconUrl], config, diff --git a/packages/prover/test/utils/e2e_env.ts b/packages/prover/test/utils/e2e_env.ts index b746b22401fa..1968fb841090 100644 --- a/packages/prover/test/utils/e2e_env.ts +++ b/packages/prover/test/utils/e2e_env.ts @@ -1,9 +1,10 @@ -import {waitForEndpoint} from "./network.js"; +import {waitForEndpoint} from "@lodestar/test-utils"; /* eslint-disable @typescript-eslint/naming-convention */ -export const rpcURL = "http://0.0.0.0:8001"; +export const rpcUrl = "http://0.0.0.0:8001"; export const beaconUrl = "http://0.0.0.0:5001"; export const proxyPort = 8888; +export const chainId = 1234; export const proxyUrl = `http://localhost:${proxyPort}`; // Wait for at least teh capella fork to be started diff --git a/packages/prover/test/utils/network.ts b/packages/prover/test/utils/network.ts deleted file mode 100644 index e439ccca5c9c..000000000000 --- a/packages/prover/test/utils/network.ts +++ /dev/null @@ -1,19 +0,0 @@ -import {request} from "node:http"; -import {sleep} from "@lodestar/utils"; - -export async function waitForEndpoint(url: string): Promise { - // eslint-disable-next-line no-constant-condition - while (true) { - const status = await new Promise((resolve) => { - const req = request(url, {method: "GET"}, (res) => { - resolve(res.statusCode); - }); - req.end(); - }); - if (status === 200) { - break; - } else { - await sleep(1000); - } - } -} diff --git a/packages/test-utils/.mocharc.yaml b/packages/test-utils/.mocharc.yaml new file mode 100644 index 000000000000..1f15bf5929e0 --- /dev/null +++ b/packages/test-utils/.mocharc.yaml @@ -0,0 +1,4 @@ +colors: true +extension: ["ts"] +node-option: + - "loader=ts-node/esm" diff --git a/packages/test-utils/.nycrc.json b/packages/test-utils/.nycrc.json new file mode 100644 index 000000000000..69aa626339a0 --- /dev/null +++ b/packages/test-utils/.nycrc.json @@ -0,0 +1,3 @@ +{ + "extends": "../../.nycrc.json" +} diff --git a/packages/test-utils/LICENSE b/packages/test-utils/LICENSE new file mode 100644 index 000000000000..f49a4e16e68b --- /dev/null +++ b/packages/test-utils/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/packages/test-utils/README.md b/packages/test-utils/README.md new file mode 100644 index 000000000000..cc7db46c0990 --- /dev/null +++ b/packages/test-utils/README.md @@ -0,0 +1,11 @@ +# lodestar-test-util + +> This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project + +Mocha / Chai and other utility to reuse across testing of other packages. + +For usage see [spec tests]("https://github.com/ChainSafe/lodestar/tree/unstable/packages/beacon-node/test/spec") + +## License + +Apache-2.0 [ChainSafe Systems](https://chainsafe.io) diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json new file mode 100644 index 000000000000..0045378f5d46 --- /dev/null +++ b/packages/test-utils/package.json @@ -0,0 +1,79 @@ +{ + "name": "@lodestar/test-utils", + "private": true, + "version": "1.9.1", + "description": "Test utilities reused across other packages", + "author": "ChainSafe Systems", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/ChainSafe/lodestar/issues" + }, + "homepage": "https://github.com/ChainSafe/lodestar#readme", + "type": "module", + "exports": { + ".": { + "import": "./lib/index.js" + }, + "./sinon": { + "import": "./lib/sinon.js" + }, + "./mocha": { + "import": "./lib/mocha.js" + } + }, + "typesVersions": { + "*": { + "*": [ + "*", + "lib/*", + "lib/*/index" + ] + } + }, + "types": "lib/index.d.ts", + "files": [ + "lib/**/*.js", + "lib/**/*.js.map", + "lib/**/*.d.ts", + "*.d.ts", + "*.js" + ], + "scripts": { + "clean": "rm -rf lib && rm -f *.tsbuildinfo", + "build": "tsc -p tsconfig.build.json", + "build:release": "yarn clean && yarn build", + "build:watch": "yarn run build --watch", + "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", + "check-types": "tsc", + "lint": "eslint --color --ext .ts src/ test/", + "lint:fix": "yarn run lint --fix", + "pretest": "yarn run check-types", + "check-readme": "typescript-docs-verifier" + }, + "repository": { + "type": "git", + "url": "git+https://github.com:ChainSafe/lodestar.git" + }, + "keywords": [ + "ethereum", + "eth-consensus", + "beacon", + "blockchain" + ], + "dependencies": { + "@lodestar/utils": "^1.9.1", + "axios": "^1.3.4", + "chai": "^4.3.7", + "mocha": "^10.2.0", + "sinon": "^15.0.3" + }, + "devDependencies": { + "@types/mocha": "^10.0.1", + "@types/yargs": "^17.0.24", + "yargs": "^17.7.1" + }, + "peerDependencies": { + "chai": "^4.3.7", + "mocha": "^10.2.0" + } +} diff --git a/packages/test-utils/src/childProcess.ts b/packages/test-utils/src/childProcess.ts new file mode 100644 index 000000000000..35149b1ca967 --- /dev/null +++ b/packages/test-utils/src/childProcess.ts @@ -0,0 +1,359 @@ +/* eslint-disable no-console */ +import childProcess from "node:child_process"; +import stream from "node:stream"; +import fs from "node:fs"; +import path from "node:path"; +import {sleep} from "@lodestar/utils"; +import {TestContext} from "./interfaces.js"; + +/** + * If timeout is greater than 0, the parent will send the signal + * identified by the killSignal property (the default is 'SIGTERM') + * if the child runs longer than timeout milliseconds. + */ +const defaultTimeout = 15 * 60 * 1000; // ms + +export type ExecChildProcessOptions = { + env?: Record; + pipeStdioToFile?: string; + pipeStdioToParent?: boolean; + logPrefix?: string; + timeoutMs?: number; + maxBuffer?: number; + signal?: AbortSignal; +}; + +/** + * Run arbitrary commands in a shell + * If the child process exits with code > 0, rejects + */ +export async function execChildProcess(cmd: string | string[], options?: ExecChildProcessOptions): Promise { + const {timeoutMs, maxBuffer, logPrefix, pipeStdioToParent, pipeStdioToFile} = options ?? {}; + const cmdStr = Array.isArray(cmd) ? cmd.join(" ") : cmd; + + return new Promise((resolve, reject) => { + const proc = childProcess.exec( + cmdStr, + {timeout: timeoutMs ?? defaultTimeout, maxBuffer, env: {...process.env, ...options?.env}}, + (err, stdout) => { + if (err) { + reject(err); + } else { + resolve(stdout.trim()); + } + } + ); + + const logPrefixStream = new stream.Transform({ + transform(chunk, _encoding, callback) { + callback(null, `${logPrefix} ${proc.pid}: ${Buffer.from(chunk).toString("utf8")}`); + }, + }); + + if (pipeStdioToParent) { + proc.stdout?.pipe(logPrefixStream).pipe(process.stdout); + proc.stderr?.pipe(logPrefixStream).pipe(process.stderr); + } + + if (pipeStdioToFile) { + fs.mkdirSync(path.dirname(pipeStdioToFile), {recursive: true}); + const stdoutFileStream = fs.createWriteStream(pipeStdioToFile); + + proc.stdout?.pipe(logPrefixStream).pipe(stdoutFileStream); + proc.stderr?.pipe(logPrefixStream).pipe(stdoutFileStream); + + proc.once("exit", (_code: number) => { + stdoutFileStream.close(); + }); + } + + if (options?.signal) { + options.signal.addEventListener( + "abort", + () => { + proc.kill("SIGKILL"); + }, + {once: true} + ); + } + }); +} + +export const stopChildProcess = async ( + childProcess: childProcess.ChildProcess, + signal: NodeJS.Signals | number = "SIGTERM" +): Promise => { + if (childProcess.killed || childProcess.exitCode !== null || childProcess.signalCode !== null) { + return; + } + + return new Promise((resolve, reject) => { + childProcess.once("error", reject); + childProcess.once("close", resolve); + childProcess.kill(signal); + }); +}; + +/** + * Gracefully stop child process by sending SIGINT signal + * + * @param childProcess - child process to gracefully stop + * @param timeoutMs - timeout to wait for child process to exit before killing + * @returns + */ +export const gracefullyStopChildProcess = async ( + childProcess: childProcess.ChildProcess, + timeoutMs = 3000 +): Promise => { + if (childProcess.killed || childProcess.exitCode !== null || childProcess.signalCode !== null) { + return; + } + + // Send signal to child process to gracefully stop + childProcess.kill("SIGINT"); + + // Wait for process to exit or timeout + const result = await Promise.race([ + new Promise((resolve) => childProcess.once("exit", resolve)).then(() => "exited"), + sleep(timeoutMs).then(() => "timeout"), + ]); + + // If process is timeout kill it + if (result === "timeout") { + await stopChildProcess(childProcess, "SIGKILL"); + } +}; + +export enum ChildProcessResolve { + /** + * Resolve immediately after spawning child process + */ + Immediate, + /** + * Resolve after child process exits + */ + Completion, + /** + * Resolve after child process is healthy. Only considered when `heath` attr is set + */ + Healthy, +} + +export type ChildProcessHealthStatus = {healthy: boolean; error?: string}; + +export type SpawnChildProcessOptions = { + /** + * Environment variables to pass to child process + */ + env?: Record; + /** + * If true, pipe child process stdio to parent process + */ + pipeStdioToFile?: string; + /** + * If true, pipe child process stdio to parent process + */ + pipeStdioToParent?: boolean; + /** + * The prefix to add to child process stdio to identify it from logs + */ + logPrefix?: string; + /** + * Hide stdio from parent process and only show errors + */ + pipeOnlyError?: boolean; + /** + * Child process resolve behavior + */ + resolveOn?: ChildProcessResolve; + /** + * Timeout to wait for child process before considering it unhealthy + */ + healthTimeoutMs?: number; + /** + * Interval to check child process health + */ + healthCheckIntervalMs?: number; + /** + * Log health checks after this time + */ + logHealthChecksAfterMs?: number; + /** + * Test context to pass to child process. Useful for testing to close the process after test case + */ + testContext?: TestContext; + /** + * Abort signal to stop child process + */ + signal?: AbortSignal; + /** + * If health attribute defined we will consider resolveOn = ChildProcessResolve.Healthy + */ + health?: () => Promise<{healthy: boolean; error?: string}>; +}; + +const defaultStartOpts = { + env: {}, + pipeStdToParent: false, + pipeOnlyError: false, + logPrefix: "", + healthCheckIntervalMs: 1000, + logHealthChecksAfterMs: 2000, + resolveOn: ChildProcessResolve.Immediate, +}; + +/** + * Spawn child process and return it + * + * @param command - command to run in child process relative to mono-repo root + * @param args - command arguments + * @param opts - options + * @returns + */ +export async function spawnChildProcess( + command: string, + args: string[], + opts?: Partial +): Promise { + const options = {...defaultStartOpts, ...opts}; + const {env, pipeStdioToFile, pipeStdioToParent, logPrefix, pipeOnlyError, signal} = options; + const {health, resolveOn, healthCheckIntervalMs, logHealthChecksAfterMs, healthTimeoutMs, testContext} = options; + + return new Promise((resolve, reject) => { + void (async () => { + const proc = childProcess.spawn(command, args, { + env: {...process.env, ...env}, + }); + + const getLogPrefixStream = (): stream.Transform => + new stream.Transform({ + transform(chunk, _encoding, callback) { + callback(null, `[${logPrefix}] [${proc.pid}]: ${Buffer.from(chunk).toString("utf8")}`); + }, + }); + + if (testContext) { + testContext.afterEach(async () => { + proc.kill("SIGINT"); + await sleep(1000, signal); + await stopChildProcess(proc); + }); + } + + if (signal) { + signal.addEventListener( + "abort", + () => { + proc.kill("SIGKILL"); + }, + {once: true} + ); + } + + if (pipeStdioToFile) { + fs.mkdirSync(path.dirname(pipeStdioToFile), {recursive: true}); + const stdoutFileStream = fs.createWriteStream(pipeStdioToFile); + + proc.stdout.pipe(getLogPrefixStream()).pipe(stdoutFileStream); + proc.stderr.pipe(getLogPrefixStream()).pipe(stdoutFileStream); + + proc.once("exit", (_code: number) => { + stdoutFileStream.close(); + }); + } + + if (pipeStdioToParent) { + proc.stdout.pipe(getLogPrefixStream()).pipe(process.stdout); + proc.stderr.pipe(getLogPrefixStream()).pipe(process.stderr); + } + + if (!pipeStdioToParent && pipeOnlyError) { + // If want to see only errors then show it on the output stream of main process + proc.stderr.pipe(getLogPrefixStream()).pipe(process.stdout); + } + + // If there is any error in running the child process, reject the promise + proc.on("error", reject); + + if (!health && resolveOn === ChildProcessResolve.Immediate) { + return resolve(proc); + } + + if (!health && resolveOn === ChildProcessResolve.Completion) { + proc.once("exit", (code: number) => { + if (code > 0) { + reject(new Error(`process exited. pid=${proc.pid}, code=${code}, command="${command} ${args.join(" ")}"`)); + } else { + resolve(proc); + } + }); + + return; + } + + // If there is a health check, wait for it to pass + if (health) { + const startHealthCheckMs = Date.now(); + const intervalId = setInterval(() => { + health() + .then((isHealthy) => { + if (isHealthy.healthy) { + clearInterval(intervalId); + clearTimeout(healthTimeoutId); + proc.removeAllListeners("exit"); + resolve(proc); + } else { + const timeSinceHealthCheckStart = Date.now() - startHealthCheckMs; + if (timeSinceHealthCheckStart > logHealthChecksAfterMs) { + console.log( + `Health check unsuccessful. logPrefix=${logPrefix} pid=${proc.pid} timeSinceHealthCheckStart=${timeSinceHealthCheckStart}` + ); + } + } + }) + .catch((e) => { + console.error("error on health check, health functions must never throw", e); + }); + }, healthCheckIntervalMs); + + const healthTimeoutId = setTimeout(() => { + clearTimeout(healthTimeoutId); + + if (intervalId !== undefined) { + reject( + new Error( + `Health check timeout. logPrefix=${logPrefix} pid=${proc.pid} healthTimeoutMs=${healthTimeoutMs}` + ) + ); + } + }, healthTimeoutMs); + + proc.once("exit", (code: number) => { + if (healthTimeoutId !== undefined) return; + + clearInterval(intervalId); + clearTimeout(healthTimeoutId); + + reject( + new Error( + `process exited before healthy. logPrefix=${logPrefix} pid=${ + proc.pid + } healthTimeoutMs=${healthTimeoutMs} code=${code} command="${command} ${args.join(" ")}"` + ) + ); + }); + } + })(); + }); +} + +export function bufferStderr(proc: childProcess.ChildProcessWithoutNullStreams): {read: () => string} { + let data = ""; + proc.stderr.on("data", (chunk) => { + data += Buffer.from(chunk).toString("utf8"); + }); + + return { + read: () => data, + }; +} diff --git a/packages/test-utils/src/cli.ts b/packages/test-utils/src/cli.ts new file mode 100644 index 000000000000..13689da5bea9 --- /dev/null +++ b/packages/test-utils/src/cli.ts @@ -0,0 +1,91 @@ +import childProcess from "node:child_process"; +import type {Argv} from "yargs"; +import {wrapTimeout} from "./timeout.js"; +import {nodeJsBinaryPath, repoRootPath} from "./path.js"; +import { + ExecChildProcessOptions, + SpawnChildProcessOptions, + execChildProcess, + spawnChildProcess, +} from "./childProcess.js"; + +// We need to make it easy for the user to pass the args for the CLI +// yargs treat `["--preset minimal"] as a single arg, so we need to split it ["--preset", "minimal"] +function parseArgs(args: string[]): string[] { + return args.map((a) => a.split(" ")).flat(); +} + +type CommandRunOptions = { + timeoutMs: number; +}; + +/** + * Run the cli command inside the main process from the Yargs object + */ +export async function runCliCommand( + cli: Argv, + args: string[], + opts: CommandRunOptions = {timeoutMs: 1000} +): Promise { + return wrapTimeout( + // eslint-disable-next-line no-async-promise-executor + new Promise(async (resolve, reject) => { + await cli.parseAsync(parseArgs(args), {}, (err, _argv, output) => { + if (err) return reject(err); + + resolve(output); + }); + }), + opts.timeoutMs + ); +} + +/** + * Exec a command in bash script mode. Useful for short-running commands + * + * @param command - The command should be relative to mono-repo root + * @param args + * @param opts + * @returns + */ +export function execCliCommand( + command: string, + args: string[], + opts?: ExecChildProcessOptions & {runWith?: "node" | "ts-node"} +): Promise { + const commandPrefixed = nodeJsBinaryPath; + + const argsPrefixed = + opts?.runWith === "ts-node" + ? // node --loader ts-node/esm cli.ts + ["--loader", "ts-node/esm", repoRootPath(command), ...args] + : // node cli.js + [repoRootPath(command), ...args]; + + return execChildProcess([commandPrefixed, ...parseArgs(argsPrefixed)], opts); +} + +/** + * Spawn a process and keep it running + * + * @param command - The command should be relative to mono-repo root + * @param args + * @param opts + * @returns + */ +export async function spawnCliCommand( + command: string, + args: string[], + opts?: SpawnChildProcessOptions & {runWith?: "node" | "ts-node"} +): Promise { + const commandPrefixed = nodeJsBinaryPath; + + const argsPrefixed = + opts?.runWith === "ts-node" + ? // node --loader ts-node/esm cli.ts + ["--loader", "ts-node/esm", repoRootPath(command), ...args] + : // node cli.js + [repoRootPath(command), ...args]; + + return spawnChildProcess(commandPrefixed, parseArgs(argsPrefixed), opts); +} diff --git a/packages/test-utils/src/http.ts b/packages/test-utils/src/http.ts new file mode 100644 index 000000000000..b4dd16390483 --- /dev/null +++ b/packages/test-utils/src/http.ts @@ -0,0 +1,55 @@ +import axios from "axios"; +import {sleep} from "@lodestar/utils"; + +type Method = "GET" | "POST" | "PUT"; + +/** + * Return the status code of a request for given url and method + */ +export async function getReqStatus(url: string, method: Method = "GET"): Promise { + const res = await axios.request({url, method}); + return res.status; +} + +/** + * Get the response body of a request for given url and method + */ +export async function getRespBody( + url: string, + method: Method = "GET", + data?: Record +): Promise { + const res = await axios.request({url, method, data}); + return res.data as T; +} + +/** + * Match the status code of a request for given url and method + */ +export async function matchReqStatus(url: string, code: number, method: Method = "GET"): Promise { + return (await getReqStatus(url, method)) === code; +} + +/** + * Match the status code of a request for given url and method + */ +export async function matchReqSuccess(url: string, method: Method = "GET"): Promise { + const status = await getReqStatus(url, method); + return status >= 200 && status < 300; +} + +/** + * Wait for a given endpoint to return a given status code + */ +export async function waitForEndpoint(url: string, statusCode = 200): Promise { + // eslint-disable-next-line no-constant-condition + while (true) { + const status = await getReqStatus(url); + + if (status === statusCode) { + break; + } + + await sleep(1000); + } +} diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts new file mode 100644 index 000000000000..84f0efe0f587 --- /dev/null +++ b/packages/test-utils/src/index.ts @@ -0,0 +1,6 @@ +export * from "./cli.js"; +export * from "./childProcess.js"; +export * from "./path.js"; +export * from "./timeout.js"; +export * from "./http.js"; +export * from "./interfaces.js"; diff --git a/packages/test-utils/src/interfaces.ts b/packages/test-utils/src/interfaces.ts new file mode 100644 index 000000000000..25e254eb28a9 --- /dev/null +++ b/packages/test-utils/src/interfaces.ts @@ -0,0 +1,5 @@ +export interface TestContext { + afterEach: (cb: () => Promise | void) => void; + beforeEach: (cb: () => Promise | void) => void; + afterAll: (cb: () => Promise | void) => void; +} diff --git a/packages/test-utils/src/mocha.ts b/packages/test-utils/src/mocha.ts new file mode 100644 index 000000000000..12c2741f67b9 --- /dev/null +++ b/packages/test-utils/src/mocha.ts @@ -0,0 +1,45 @@ +import type {Suite} from "mocha"; +import {TestContext} from "./interfaces.js"; +export {TestContext} from "./interfaces.js"; + +/** + * Create a Mocha context object that can be used to register callbacks that will be executed + */ +export function getMochaContext(suite: Suite): TestContext { + const afterEachCallbacks: (() => Promise | void)[] = []; + const beforeEachCallbacks: (() => Promise | void)[] = []; + const afterAllCallbacks: (() => Promise | void)[] = []; + + const context: TestContext = { + afterEach: (cb) => afterEachCallbacks.push(cb), + beforeEach: (cb) => beforeEachCallbacks.push(cb), + afterAll: (cb) => afterAllCallbacks.push(cb), + }; + + const callbacks = [afterEachCallbacks, beforeEachCallbacks, afterAllCallbacks]; + const hooks = [suite.afterEach, suite.beforeEach, suite.afterAll]; + + for (const [index, cbs] of callbacks.entries()) { + const hook = hooks[index].bind(suite); + + hook(async function mochaHook() { + // Add increased timeout for that hook + this.timeout(10000); + + const errs: Error[] = []; + for (const cb of cbs) { + try { + await cb(); + } catch (e) { + errs.push(e as Error); + } + } + cbs.length = 0; // Reset array + if (errs.length > 0) { + throw errs[0]; + } + }); + } + + return context; +} diff --git a/packages/test-utils/src/path.ts b/packages/test-utils/src/path.ts new file mode 100644 index 000000000000..24b10d272e95 --- /dev/null +++ b/packages/test-utils/src/path.ts @@ -0,0 +1,21 @@ +import path from "node:path"; + +/** + * Return the absolute path to a file relative to the current file + * From https://blog.logrocket.com/alternatives-dirname-node-js-es-modules + */ +export function esmRelativePathResolve(relativePath: string): string { + return new URL(relativePath, import.meta.url).toString().replace(/^file:\/\//, ""); +} + +/** + * Return the path to the root of the repo + */ +export function repoRootPath(fileDirPath: string): string { + return path.join(esmRelativePathResolve("../../../"), fileDirPath); +} + +/** + * Path to the node binary + */ +export const nodeJsBinaryPath = "node"; diff --git a/packages/test-utils/src/sinon.ts b/packages/test-utils/src/sinon.ts new file mode 100644 index 000000000000..9c75dd171248 --- /dev/null +++ b/packages/test-utils/src/sinon.ts @@ -0,0 +1,23 @@ +import {SinonSpy, spy} from "sinon"; + +type Callback = () => void; +type Handler = (cb: Callback) => void; + +/** + * Stub the logger methods + */ +export function stubLogger(context: {beforeEach: Handler; afterEach: Handler}, logger = console): void { + context.beforeEach(() => { + spy(logger, "info"); + spy(logger, "log"); + spy(logger, "warn"); + spy(logger, "error"); + }); + + context.afterEach(() => { + (logger.info as SinonSpy).restore(); + (logger.log as SinonSpy).restore(); + (logger.warn as SinonSpy).restore(); + (logger.error as SinonSpy).restore(); + }); +} diff --git a/packages/test-utils/src/timeout.ts b/packages/test-utils/src/timeout.ts new file mode 100644 index 000000000000..77e716c0691c --- /dev/null +++ b/packages/test-utils/src/timeout.ts @@ -0,0 +1,17 @@ +import {sleep} from "@lodestar/utils"; + +/** + * Wrap a promise with a timeout + */ +export function wrapTimeout( + p: Promise, + timeoutMs: number, + opts?: {timeoutMsg?: string; signal?: AbortSignal} +): Promise { + return Promise.race([ + p, + sleep(timeoutMs, opts?.signal).then(() => { + throw new Error(opts?.timeoutMsg ?? `Promise timeout after ${timeoutMs}ms.`); + }), + ]) as Promise; +} diff --git a/packages/test-utils/tsconfig.build.json b/packages/test-utils/tsconfig.build.json new file mode 100644 index 000000000000..3c66d5619616 --- /dev/null +++ b/packages/test-utils/tsconfig.build.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.build.json", + "include": ["src"], + "compilerOptions": { + "outDir": "./lib", + "typeRoots": ["../../node_modules/@types", "./node_modules/@types", "../../types"] + } +} diff --git a/packages/test-utils/tsconfig.json b/packages/test-utils/tsconfig.json new file mode 100644 index 000000000000..f81823701532 --- /dev/null +++ b/packages/test-utils/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "typeRoots": ["../../node_modules/@types", "./node_modules/@types", "../../types"] + } +} From b5960007d600a93eff49b295b1b1b1559e3d707f Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 10 Jul 2023 15:54:44 +0200 Subject: [PATCH 49/96] chore: misc dashboards update (#5738) * Update dashboards * Use rate function * Ensure to always use variables to match job names * Run scripts/lint-grafana-dashboards.mjs * Do not reassign target.expr * Fix updating interval and job name matchers * Run scripts/lint-grafana-dashboards.mjs * Add rationale * Update dashboards/lodestar_networking.json --------- Co-authored-by: Nico Flaig --- dashboards/lodestar_block_processor.json | 419 +----- dashboards/lodestar_block_production.json | 102 +- dashboards/lodestar_bls_thread_pool.json | 161 +-- dashboards/lodestar_debug_gossipsub.json | 1459 +++----------------- dashboards/lodestar_discv5.json | 309 +---- dashboards/lodestar_execution_engine.json | 526 +------ dashboards/lodestar_libp2p.json | 140 +- dashboards/lodestar_multinode.json | 125 +- dashboards/lodestar_networking.json | 1352 +++++------------- dashboards/lodestar_rest_api.json | 84 +- dashboards/lodestar_state_cache_regen.json | 346 +---- dashboards/lodestar_summary.json | 411 +----- dashboards/lodestar_sync.json | 234 +--- dashboards/lodestar_validator_client.json | 243 +--- dashboards/lodestar_validator_monitor.json | 775 ++++++----- dashboards/lodestar_vm_host.json | 498 +------ scripts/download_dashboards.mjs | 0 scripts/grafana_push_dashboards.sh | 10 +- scripts/lint-grafana-dashboard.mjs | 38 +- 19 files changed, 1223 insertions(+), 6009 deletions(-) mode change 100644 => 100755 scripts/download_dashboards.mjs diff --git a/dashboards/lodestar_block_processor.json b/dashboards/lodestar_block_processor.json index 6b3f57c0ecb9..732b558d79c0 100644 --- a/dashboards/lodestar_block_processor.json +++ b/dashboards/lodestar_block_processor.json @@ -1,19 +1,19 @@ { "__inputs": [ { - "description": "", - "label": "Prometheus", "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "description": "", "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" + "pluginName": "Prometheus" }, { - "description": "", - "label": "Beacon node job name", "name": "VAR_BEACON_JOB", "type": "constant", - "value": "beacon" + "label": "Beacon node job name", + "value": "beacon", + "description": "" } ], "annotations": { @@ -127,19 +127,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -222,19 +209,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -319,19 +293,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -363,7 +324,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_block_processor_queue_job_time_seconds_sum[$rate_interval])/delta(lodestar_block_processor_queue_job_time_seconds_count[$rate_interval])", + "expr": "rate(lodestar_block_processor_queue_job_time_seconds_sum[$rate_interval])/rate(lodestar_block_processor_queue_job_time_seconds_count[$rate_interval])", "instant": false, "interval": "", "legendFormat": "block_processor", @@ -415,19 +376,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -459,7 +407,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_block_processor_queue_dropped_jobs_total[$rate_interval])/(delta(lodestar_block_processor_queue_job_time_seconds_count[$rate_interval])+delta(lodestar_block_processor_queue_dropped_jobs_total[$rate_interval]))", + "expr": "rate(lodestar_block_processor_queue_dropped_jobs_total[$rate_interval])/(rate(lodestar_block_processor_queue_job_time_seconds_count[$rate_interval])+rate(lodestar_block_processor_queue_dropped_jobs_total[$rate_interval]))", "instant": false, "interval": "", "legendFormat": "block_processor", @@ -511,19 +459,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -555,7 +490,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_block_processor_queue_job_wait_time_seconds_sum[$rate_interval])/delta(lodestar_block_processor_queue_job_wait_time_seconds_count[$rate_interval])", + "expr": "rate(lodestar_block_processor_queue_job_wait_time_seconds_sum[$rate_interval])/rate(lodestar_block_processor_queue_job_wait_time_seconds_count[$rate_interval])", "instant": false, "interval": "", "legendFormat": "block_processor", @@ -607,19 +542,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -701,20 +623,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -929,7 +838,7 @@ "mode": "scheme", "reverse": false, "scale": "exponential", - "scheme": "Oranges", + "scheme": "Magma", "steps": 64 }, "exemplars": { @@ -962,7 +871,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "rate(lodestar_stfn_epoch_transition_seconds_bucket[$rate_interval])", + "expr": "32*12*rate(lodestar_stfn_epoch_transition_seconds_bucket[$rate_interval])", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", @@ -1009,7 +918,7 @@ "mode": "scheme", "reverse": false, "scale": "exponential", - "scheme": "Oranges", + "scheme": "Magma", "steps": 64 }, "exemplars": { @@ -1042,7 +951,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "rate(lodestar_stfn_process_block_seconds_bucket[$rate_interval])", + "expr": "12*rate(lodestar_stfn_process_block_seconds_bucket[$rate_interval])", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", @@ -1095,10 +1004,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, "unit": "s" }, "overrides": [] @@ -1184,10 +1089,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, "unit": "s" }, "overrides": [] @@ -1272,10 +1173,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, "unit": "s" }, "overrides": [] @@ -1309,7 +1206,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "delta(lodestar_stfn_epoch_transition_commit_seconds_sum[$rate_interval])\n/\ndelta(lodestar_stfn_epoch_transition_commit_seconds_count[$rate_interval])", + "expr": "rate(lodestar_stfn_epoch_transition_commit_seconds_sum[$rate_interval])\n/\nrate(lodestar_stfn_epoch_transition_commit_seconds_count[$rate_interval])", "interval": "", "legendFormat": "epoch transition", "range": true, @@ -1361,10 +1258,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, "unit": "s" }, "overrides": [] @@ -1449,10 +1342,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, "unit": "percentunit" }, "overrides": [ @@ -1552,10 +1441,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, "unit": "percentunit" }, "overrides": [ @@ -1656,10 +1541,6 @@ }, "mappings": [], "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [] - }, "unit": "none" }, "overrides": [ @@ -1759,10 +1640,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, "unit": "none" }, "overrides": [ @@ -1862,10 +1739,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, "unit": "s" }, "overrides": [] @@ -1950,10 +1823,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, "unit": "short" }, "overrides": [] @@ -2137,18 +2006,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -2228,19 +2085,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2322,18 +2167,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -2368,7 +2201,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(delta(lodestar_bls_thread_pool_time_seconds_sum[$rate_interval]))/sum(delta(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval]))", + "expr": "sum(rate(lodestar_bls_thread_pool_time_seconds_sum[$rate_interval]))/sum(rate(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval]))", "interval": "", "legendFormat": "pool", "refId": "A" @@ -2420,18 +2253,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -2518,18 +2339,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -2616,18 +2425,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -2662,7 +2459,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_bls_thread_pool_queue_job_wait_time_seconds_sum[$rate_interval])/delta(lodestar_bls_thread_pool_queue_job_wait_time_seconds_count[$rate_interval])", + "expr": "rate(lodestar_bls_thread_pool_queue_job_wait_time_seconds_sum[$rate_interval])/rate(lodestar_bls_thread_pool_queue_job_wait_time_seconds_count[$rate_interval])", "interval": "", "legendFormat": "pool", "refId": "A" @@ -2715,18 +2512,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -2759,7 +2544,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_bls_thread_pool_latency_to_worker_sum[6m])/delta(lodestar_bls_thread_pool_latency_to_worker_count[6m])", + "expr": "rate(lodestar_bls_thread_pool_latency_to_worker_sum[6m])/rate(lodestar_bls_thread_pool_latency_to_worker_count[6m])", "interval": "", "legendFormat": "to worker", "refId": "to worker" @@ -2770,7 +2555,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_bls_thread_pool_latency_from_worker_sum[6m])/delta(lodestar_bls_thread_pool_latency_from_worker_count[6m])", + "expr": "rate(lodestar_bls_thread_pool_latency_from_worker_sum[6m])/rate(lodestar_bls_thread_pool_latency_from_worker_count[6m])", "hide": false, "interval": "", "legendFormat": "from worker", @@ -2823,18 +2608,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -2869,7 +2642,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_bls_thread_pool_batch_sigs_success_total[$rate_interval])/delta(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])", + "expr": "rate(lodestar_bls_thread_pool_batch_sigs_success_total[$rate_interval])/rate(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])", "interval": "", "legendFormat": "pool", "refId": "A" @@ -2921,18 +2694,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -2964,7 +2725,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_bls_thread_pool_sig_sets_started_total[$rate_interval])/(delta(lodestar_bls_thread_pool_jobs_started_total[$rate_interval])>0)", + "expr": "rate(lodestar_bls_thread_pool_sig_sets_started_total[$rate_interval])/(rate(lodestar_bls_thread_pool_jobs_started_total[$rate_interval])>0)", "interval": "", "legendFormat": "pool", "refId": "A" @@ -3015,18 +2776,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [ @@ -3074,7 +2823,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_bls_thread_pool_error_jobs_signature_sets_count[$rate_interval])\n/\n(\n delta(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])\n + delta(lodestar_bls_thread_pool_error_jobs_signature_sets_count[$rate_interval])\n)", + "expr": "rate(lodestar_bls_thread_pool_error_jobs_signature_sets_count[$rate_interval])\n/\n(\n rate(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])\n + rate(lodestar_bls_thread_pool_error_jobs_signature_sets_count[$rate_interval])\n)", "interval": "", "legendFormat": "sig_error_rate", "refId": "A" @@ -3138,18 +2887,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -3181,7 +2918,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_bls_thread_pool_jobs_started_total[$rate_interval])/delta(lodestar_bls_thread_pool_job_groups_started_total[$rate_interval])", + "expr": "rate(lodestar_bls_thread_pool_jobs_started_total[$rate_interval])/rate(lodestar_bls_thread_pool_job_groups_started_total[$rate_interval])", "interval": "", "legendFormat": "pool", "refId": "A" @@ -3256,19 +2993,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -3379,19 +3104,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -3498,19 +3211,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3601,19 +3302,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3721,18 +3410,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -3766,7 +3443,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(beacon_fork_choice_find_head_seconds_sum[$rate_interval])/delta(beacon_fork_choice_find_head_seconds_count[$rate_interval])", + "expr": "rate(beacon_fork_choice_find_head_seconds_sum[$rate_interval])/rate(beacon_fork_choice_find_head_seconds_count[$rate_interval])", "interval": "", "legendFormat": "find head", "refId": "A" @@ -3816,19 +3493,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -3928,19 +3593,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -4068,14 +3721,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, "unit": "percentunit" }, "overrides": [ diff --git a/dashboards/lodestar_block_production.json b/dashboards/lodestar_block_production.json index ae0d7c5fadb9..f0d87d29e20a 100644 --- a/dashboards/lodestar_block_production.json +++ b/dashboards/lodestar_block_production.json @@ -119,20 +119,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -313,19 +300,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -406,20 +380,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -609,20 +570,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -971,16 +919,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1077,16 +1016,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1189,16 +1119,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1247,16 +1168,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, diff --git a/dashboards/lodestar_bls_thread_pool.json b/dashboards/lodestar_bls_thread_pool.json index 343573b6b670..a8021ace1102 100644 --- a/dashboards/lodestar_bls_thread_pool.json +++ b/dashboards/lodestar_bls_thread_pool.json @@ -124,19 +124,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -205,20 +192,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -293,19 +267,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -335,7 +296,7 @@ "pluginVersion": "7.4.5", "targets": [ { - "expr": "sum(delta(lodestar_bls_thread_pool_time_seconds_sum[$rate_interval]))/sum(delta(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval]))", + "expr": "sum(rate(lodestar_bls_thread_pool_time_seconds_sum[$rate_interval]))/sum(rate(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval]))", "interval": "", "legendFormat": "pool", "refId": "A" @@ -381,19 +342,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -469,19 +417,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -557,19 +492,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -599,7 +521,7 @@ "pluginVersion": "7.4.5", "targets": [ { - "expr": "delta(lodestar_bls_thread_pool_queue_job_wait_time_seconds_sum[$rate_interval])/delta(lodestar_bls_thread_pool_queue_job_wait_time_seconds_count[$rate_interval])", + "expr": "rate(lodestar_bls_thread_pool_queue_job_wait_time_seconds_sum[$rate_interval])/rate(lodestar_bls_thread_pool_queue_job_wait_time_seconds_count[$rate_interval])", "interval": "", "legendFormat": "pool", "refId": "A" @@ -646,19 +568,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -690,7 +599,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_bls_thread_pool_latency_to_worker_sum[6m])/delta(lodestar_bls_thread_pool_latency_to_worker_count[6m])", + "expr": "rate(lodestar_bls_thread_pool_latency_to_worker_sum[6m])/rate(lodestar_bls_thread_pool_latency_to_worker_count[6m])", "interval": "", "legendFormat": "to worker", "refId": "to worker" @@ -701,7 +610,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_bls_thread_pool_latency_from_worker_sum[6m])/delta(lodestar_bls_thread_pool_latency_from_worker_count[6m])", + "expr": "rate(lodestar_bls_thread_pool_latency_from_worker_sum[6m])/rate(lodestar_bls_thread_pool_latency_from_worker_count[6m])", "hide": false, "interval": "", "legendFormat": "from worker", @@ -748,19 +657,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -790,7 +686,7 @@ "pluginVersion": "7.4.5", "targets": [ { - "expr": "delta(lodestar_bls_thread_pool_batch_sigs_success_total[$rate_interval])/delta(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])", + "expr": "rate(lodestar_bls_thread_pool_batch_sigs_success_total[$rate_interval])/rate(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])", "interval": "", "legendFormat": "pool", "refId": "A" @@ -836,19 +732,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -875,7 +758,7 @@ "pluginVersion": "7.4.5", "targets": [ { - "expr": "delta(lodestar_bls_thread_pool_sig_sets_started_total[$rate_interval])/(delta(lodestar_bls_thread_pool_jobs_started_total[$rate_interval])>0)", + "expr": "rate(lodestar_bls_thread_pool_sig_sets_started_total[$rate_interval])/(rate(lodestar_bls_thread_pool_jobs_started_total[$rate_interval])>0)", "interval": "", "legendFormat": "pool", "refId": "A" @@ -920,19 +803,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [ @@ -979,7 +849,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_bls_thread_pool_error_jobs_signature_sets_count[$rate_interval])\n/\n(\n delta(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])\n + delta(lodestar_bls_thread_pool_error_jobs_signature_sets_count[$rate_interval])\n)", + "expr": "rate(lodestar_bls_thread_pool_error_jobs_signature_sets_count[$rate_interval])\n/\n(\n rate(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])\n + rate(lodestar_bls_thread_pool_error_jobs_signature_sets_count[$rate_interval])\n)", "interval": "", "legendFormat": "sig_error_rate", "refId": "A" @@ -1037,19 +907,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -1076,7 +933,7 @@ "pluginVersion": "7.4.5", "targets": [ { - "expr": "delta(lodestar_bls_thread_pool_jobs_started_total[$rate_interval])/delta(lodestar_bls_thread_pool_job_groups_started_total[$rate_interval])", + "expr": "rate(lodestar_bls_thread_pool_jobs_started_total[$rate_interval])/rate(lodestar_bls_thread_pool_job_groups_started_total[$rate_interval])", "interval": "", "legendFormat": "pool", "refId": "A" diff --git a/dashboards/lodestar_debug_gossipsub.json b/dashboards/lodestar_debug_gossipsub.json index b20b7e62293a..0bee77fed589 100644 --- a/dashboards/lodestar_debug_gossipsub.json +++ b/dashboards/lodestar_debug_gossipsub.json @@ -1,12 +1,12 @@ { "__inputs": [ { - "description": "", - "label": "Prometheus", "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "description": "", "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" + "pluginName": "Prometheus" } ], "annotations": { @@ -119,16 +119,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [ { @@ -183,11 +174,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "lodestar_gossip_peer_score_by_threshold_count", + "expr": "avg_over_time( lodestar_gossip_peer_score_by_threshold_count [$rate_interval])", "format": "time_series", "interval": "", "legendFormat": "{{threshold}}", + "range": true, "refId": "A" } ], @@ -201,16 +194,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -279,15 +263,6 @@ "type": "value" } ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "none" }, "overrides": [] @@ -340,20 +315,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -408,11 +370,7 @@ "fixedColor": "green", "mode": "fixed" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - } + "mappings": [] }, "overrides": [] }, @@ -467,20 +425,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -565,19 +510,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "locale" }, "overrides": [ @@ -787,16 +719,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -826,11 +749,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "lodestar_gossip_mesh_peers_by_type_count{type!~\"beacon_attestation\"}", + "expr": "avg_over_time(lodestar_gossip_mesh_peers_by_type_count{type!~\"beacon_attestation\"}[$rate_interval])", "format": "time_series", "interval": "", "legendFormat": "{{type}}", + "range": true, "refId": "A" } ], @@ -877,16 +802,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [ { @@ -1057,16 +973,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1107,54 +1014,71 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], "unit": "none" }, "overrides": [] }, - "fill": 1, - "fillGradient": 1, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, - "hiddenSeries": false, "id": 80, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, "pluginVersion": "9.3.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { @@ -1167,35 +1091,8 @@ "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Gossip validation queue - Processed jobs / slot", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "collapsed": false, @@ -1224,96 +1121,86 @@ "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], "unit": "s" }, "overrides": [] }, - "fill": 1, - "fillGradient": 1, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, - "hiddenSeries": false, "id": 82, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, "pluginVersion": "9.3.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_gossip_validation_queue_job_wait_time_seconds_sum[$rate_interval])/delta(lodestar_gossip_validation_queue_job_wait_time_seconds_count[$rate_interval])", + "expr": "rate(lodestar_gossip_validation_queue_job_wait_time_seconds_sum[$rate_interval])/rate(lodestar_gossip_validation_queue_job_wait_time_seconds_count[$rate_interval])", "format": "time_series", "interval": "", "legendFormat": "{{topic}}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Gossip validation queues - Job wait time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "datasource": { @@ -1357,19 +1244,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1401,7 +1275,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_gossip_validation_queue_dropped_jobs_total[$rate_interval])/(delta(lodestar_gossip_validation_queue_job_time_seconds_count[$rate_interval])+delta(lodestar_gossip_validation_queue_dropped_jobs_total[$rate_interval]))", + "expr": "rate(lodestar_gossip_validation_queue_dropped_jobs_total[$rate_interval])/(rate(lodestar_gossip_validation_queue_job_time_seconds_count[$rate_interval])+rate(lodestar_gossip_validation_queue_dropped_jobs_total[$rate_interval]))", "interval": "", "legendFormat": "{{topic}}", "refId": "A" @@ -1453,19 +1327,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -1496,7 +1357,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_gossip_block_elapsed_time_till_received_sum[$rate_interval])\n/\ndelta(lodestar_gossip_block_elapsed_time_till_received_count[$rate_interval])", + "expr": "rate(lodestar_gossip_block_elapsed_time_till_received_sum[$rate_interval])\n/\nrate(lodestar_gossip_block_elapsed_time_till_received_count[$rate_interval])", "interval": "", "legendFormat": "till received", "refId": "A" @@ -1507,7 +1368,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_gossip_block_elapsed_time_till_processed_sum[$rate_interval])\n/\ndelta(lodestar_gossip_block_elapsed_time_till_processed_count[$rate_interval])", + "expr": "rate(lodestar_gossip_block_elapsed_time_till_processed_sum[$rate_interval])\n/\nrate(lodestar_gossip_block_elapsed_time_till_processed_count[$rate_interval])", "hide": false, "interval": "", "legendFormat": "till processed", @@ -1518,95 +1379,87 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], "unit": "none" }, "overrides": [] }, - "fill": 1, - "fillGradient": 1, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, - "hiddenSeries": false, "id": 88, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, "pluginVersion": "9.3.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "lodestar_gossip_validation_queue_length", + "editorMode": "code", + "expr": "avg_over_time(\n lodestar_gossip_validation_queue_length\n[$rate_interval])", "interval": "", "legendFormat": "{{topic}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Gossip validation queues - Length", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "collapsed": false, @@ -1674,20 +1527,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1766,20 +1606,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1808,10 +1635,12 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "gossipsub_mesh_peer_count", + "expr": "avg_over_time( gossipsub_mesh_peer_count [$rate_interval])", "interval": "", "legendFormat": "{{topicStr}}", + "range": true, "refId": "A" } ], @@ -1859,19 +1688,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -1964,19 +1780,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -2096,19 +1899,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -2188,20 +1978,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2230,14 +2007,16 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "(sum(rate(gossipsub_msg_received_status_total{status=\"duplicate\"} [32m])) by (topic))\n/\n(sum(rate(gossipsub_msg_received_status_total{status=\"valid\"} [32m])) by (topic))", + "expr": "(sum(rate(gossipsub_msg_received_status_total{status=\"duplicate\",topic!~\"light.*\"} [32m])) by (topic))\n/\n(sum(rate(gossipsub_msg_received_status_total{status=\"valid\",topic!~\"light.*\"} [32m])) by (topic))", "interval": "", "legendFormat": "{{topic}} {{status}}", + "range": true, "refId": "A" } ], - "title": "Received msg duplicate / valid rate", + "title": "Received msg duplicate / valid rate (except light_* topics)", "type": "timeseries" }, { @@ -2282,19 +2061,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -2376,19 +2142,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -2470,19 +2223,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -2564,19 +2304,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -2658,19 +2385,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -2764,19 +2478,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -2902,20 +2603,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2995,19 +2683,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "Bps" }, "overrides": [] @@ -3089,19 +2764,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -3182,19 +2844,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -3275,20 +2924,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3368,19 +3004,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -3486,20 +3109,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3578,16 +3188,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [ { @@ -3758,20 +3359,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3850,20 +3438,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3943,20 +3518,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -4036,20 +3598,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -4128,20 +3677,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -4170,10 +3706,12 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "gossipsub_score_per_mesh_avg", + "expr": "avg_over_time( gossipsub_score_per_mesh_avg [$rate_interval])", "interval": "", "legendFormat": "{{topic}}", + "range": true, "refId": "A" } ], @@ -4220,20 +3758,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -4262,10 +3787,12 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "gossipsub_score_per_mesh_min", + "expr": "avg_over_time( gossipsub_score_per_mesh_min [$rate_interval])", "interval": "", "legendFormat": "{{topic}}", + "range": true, "refId": "A" } ], @@ -4364,98 +3891,23 @@ }, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "max" - }, - "properties": [ - { - "id": "custom.fillBelowTo", - "value": "avg" - }, - { - "id": "custom.fillOpacity", - "value": 12 - }, - { - "id": "color", - "value": { - "fixedColor": "semi-dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "avg" - }, - "properties": [ - { - "id": "custom.fillBelowTo", - "value": "min" - }, - { - "id": "custom.fillOpacity", - "value": 12 - }, - { - "id": "custom.gradientMode", - "value": "opacity" - }, - { - "id": "color", - "value": { - "fixedColor": "super-light-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "min" + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [] }, "gridPos": { "h": 8, @@ -4534,16 +3986,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [ { @@ -4690,20 +4133,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -4783,19 +4213,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -4876,19 +4293,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -5176,16 +4580,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [ { @@ -5357,19 +4752,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -5449,16 +4831,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [ { @@ -5627,19 +5000,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [ @@ -5765,19 +5125,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -5918,15 +5265,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "none" }, "overrides": [] @@ -6154,19 +5492,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [ @@ -6324,19 +5649,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [ @@ -6415,7 +5727,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "1 - sum(rate(gossipsub_iwant_promise_delivery_seconds_count[$__rate_interval])) / sum(rate(gossipsub_iwant_promise_sent_total[$__rate_interval]))", + "expr": "1 - sum(rate(gossipsub_iwant_promise_delivery_seconds_count[$rate_interval])) / sum(rate(gossipsub_iwant_promise_sent_total[$rate_interval]))", "hide": false, "interval": "", "legendFormat": "never delivered", @@ -6481,19 +5793,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -6741,19 +6040,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -6883,19 +6169,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -7051,19 +6324,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -7144,19 +6404,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [ @@ -7262,19 +6509,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -7355,19 +6589,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -7448,19 +6669,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -7541,19 +6749,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [ @@ -7720,19 +6915,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -7838,19 +7020,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -7944,19 +7113,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -8039,19 +7195,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -8132,19 +7275,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [ @@ -8267,19 +7397,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -8430,19 +7547,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "Bps" }, "overrides": [] @@ -8535,19 +7639,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -8640,19 +7731,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -8745,19 +7823,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -8850,19 +7915,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -8955,19 +8007,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -9060,19 +8099,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -9165,19 +8191,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -9270,19 +8283,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -9375,19 +8375,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -9480,19 +8467,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -9585,19 +8559,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] diff --git a/dashboards/lodestar_discv5.json b/dashboards/lodestar_discv5.json index 784b872af3f4..02c0a3b38956 100644 --- a/dashboards/lodestar_discv5.json +++ b/dashboards/lodestar_discv5.json @@ -107,20 +107,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -201,19 +188,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -295,19 +269,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -390,19 +351,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -485,19 +433,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -579,19 +514,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -674,19 +596,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -768,19 +677,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -863,19 +759,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -957,19 +840,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1064,19 +934,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1159,19 +1016,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1255,19 +1099,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [ @@ -1381,19 +1212,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1476,19 +1294,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1571,19 +1376,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1678,19 +1470,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "decbytes" }, "overrides": [] @@ -1796,19 +1575,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1889,20 +1655,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -2018,20 +1771,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2111,19 +1851,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -2154,7 +1881,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "rate(discv5_worker_nodejs_gc_duration_seconds_sum[$__rate_interval])", + "expr": "rate(discv5_worker_nodejs_gc_duration_seconds_sum[$rate_interval])", "legendFormat": "{{kind}}", "range": true, "refId": "A" @@ -2203,20 +1930,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2296,19 +2010,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] diff --git a/dashboards/lodestar_execution_engine.json b/dashboards/lodestar_execution_engine.json index 698480f19355..cd72b712f9a8 100644 --- a/dashboards/lodestar_execution_engine.json +++ b/dashboards/lodestar_execution_engine.json @@ -120,19 +120,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -216,19 +203,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -313,19 +287,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -358,7 +319,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_engine_http_processor_queue_job_time_seconds_sum[$rate_interval])/delta(lodestar_engine_http_processor_queue_job_time_seconds_count[$rate_interval])", + "expr": "rate(lodestar_engine_http_processor_queue_job_time_seconds_sum[$rate_interval])/rate(lodestar_engine_http_processor_queue_job_time_seconds_count[$rate_interval])", "instant": false, "interval": "", "legendFormat": "engine_processor", @@ -410,19 +371,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -455,7 +403,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_engine_http_processor_queue_dropped_jobs_total[$rate_interval])/(delta(lodestar_engine_http_processor_queue_job_time_seconds_count[$rate_interval])+delta(lodestar_engine_http_processor_queue_dropped_jobs_total[$rate_interval]))", + "expr": "rate(lodestar_engine_http_processor_queue_dropped_jobs_total[$rate_interval])/(rate(lodestar_engine_http_processor_queue_job_time_seconds_count[$rate_interval])+rate(lodestar_engine_http_processor_queue_dropped_jobs_total[$rate_interval]))", "instant": false, "interval": "", "legendFormat": "engine_processor", @@ -507,19 +455,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -552,7 +487,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(lodestar_engine_http_processor_queue_job_wait_time_seconds_sum[$rate_interval])/delta(lodestar_engine_http_processor_queue_job_wait_time_seconds_count[$rate_interval])", + "expr": "rate(lodestar_engine_http_processor_queue_job_wait_time_seconds_sum[$rate_interval])/rate(lodestar_engine_http_processor_queue_job_wait_time_seconds_count[$rate_interval])", "instant": false, "interval": "", "legendFormat": "engine_processor", @@ -604,19 +539,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -727,19 +649,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -790,16 +699,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -885,19 +785,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -981,19 +868,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1077,19 +951,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -1174,19 +1035,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1281,19 +1129,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1373,20 +1208,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1479,19 +1301,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "µs" }, "overrides": [ @@ -1597,19 +1406,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "µs" }, "overrides": [ @@ -1715,19 +1511,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "µs" }, "overrides": [ @@ -1860,19 +1643,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1960,15 +1730,6 @@ "type": "value" } ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "none" }, "overrides": [] @@ -2019,16 +1780,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2080,15 +1832,6 @@ "fieldConfig": { "defaults": { "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "s" }, "overrides": [] @@ -2141,15 +1884,6 @@ "fieldConfig": { "defaults": { "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "dateTimeFromNow" }, "overrides": [] @@ -2238,19 +1972,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -2358,19 +2079,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -2455,19 +2163,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -2574,19 +2269,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -2669,19 +2351,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -2791,15 +2460,6 @@ "type": "value" } ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "none" }, "overrides": [] @@ -2854,20 +2514,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2923,19 +2570,6 @@ "mode": "thresholds" }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -2991,19 +2625,6 @@ "mode": "thresholds" }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "dateTimeAsSystem" }, "overrides": [] @@ -3056,15 +2677,6 @@ "fieldConfig": { "defaults": { "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "none" }, "overrides": [] @@ -3119,20 +2731,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3187,20 +2786,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3252,16 +2838,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3312,16 +2889,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3373,15 +2941,6 @@ "fieldConfig": { "defaults": { "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "dateTimeFromNow" }, "overrides": [] @@ -3433,16 +2992,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3493,16 +3043,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3556,20 +3097,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3656,19 +3184,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -3764,19 +3279,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] diff --git a/dashboards/lodestar_libp2p.json b/dashboards/lodestar_libp2p.json index bd3480e698b0..08f500cf5a40 100644 --- a/dashboards/lodestar_libp2p.json +++ b/dashboards/lodestar_libp2p.json @@ -120,19 +120,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "Bps" }, "overrides": [] @@ -213,19 +200,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "Bps" }, "overrides": [] @@ -331,20 +305,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -423,20 +384,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -533,19 +481,6 @@ ], "max": 2, "min": -1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -626,19 +561,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "Bps" }, "overrides": [] @@ -719,19 +641,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "Bps" }, "overrides": [] @@ -837,20 +746,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -929,20 +825,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1021,20 +904,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, diff --git a/dashboards/lodestar_multinode.json b/dashboards/lodestar_multinode.json index d8883f57b070..ac710364985b 100644 --- a/dashboards/lodestar_multinode.json +++ b/dashboards/lodestar_multinode.json @@ -72,20 +72,7 @@ }, "type": "value" } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + ] }, "overrides": [] }, @@ -165,20 +152,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -249,20 +223,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -334,19 +295,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "decbytes" }, "overrides": [] @@ -420,19 +368,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -461,7 +396,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(process_cpu_seconds_total[$__rate_interval])", + "expr": "rate(process_cpu_seconds_total[$rate_interval])", "interval": "", "intervalFactor": 1, "legendFormat": "{{job}}", @@ -477,16 +412,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -552,16 +478,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -642,20 +559,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -728,19 +632,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -769,7 +660,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum by (job) (\n rate(nodejs_gc_pause_seconds_total[$__rate_interval])\n)", + "expr": "sum by (job) (\n rate(nodejs_gc_pause_seconds_total[$rate_interval])\n)", "interval": "", "intervalFactor": 1, "legendFormat": "{{job}}", diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index 461e651f8f5f..8fb72f76ef1c 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -1,12 +1,12 @@ { "__inputs": [ { - "description": "", - "label": "Prometheus", "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "description": "", "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" + "pluginName": "Prometheus" } ], "annotations": { @@ -122,19 +122,6 @@ }, "mappings": [], "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -218,16 +205,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -308,16 +286,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -398,16 +367,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [ { @@ -550,16 +510,7 @@ }, "decimals": 0, "mappings": [], - "min": 0.01, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "min": 0.01 }, "overrides": [] }, @@ -613,16 +564,7 @@ "mode": "thresholds" }, "mappings": [], - "min": 0.01, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "min": 0.01 }, "overrides": [] }, @@ -705,16 +647,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -794,16 +727,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -913,19 +837,6 @@ }, "mappings": [], "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -957,7 +868,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_peer_goodbye_received_total[$rate_interval])>0", + "expr": "rate(lodestar_peer_goodbye_received_total[$rate_interval])>0", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1011,19 +922,6 @@ }, "mappings": [], "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1112,19 +1010,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1156,7 +1041,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_peer_goodbye_sent_total[$rate_interval])>0", + "expr": "rate(lodestar_peer_goodbye_sent_total[$rate_interval])>0", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1322,19 +1207,6 @@ }, "mappings": [], "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1644,19 +1516,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [ @@ -1771,19 +1630,6 @@ }, "links": [], "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -1867,19 +1713,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -2003,10 +1836,12 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "increase(lodestar_peers_requested_total_to_connect[$rate_interval])", + "expr": "60*rate(lodestar_peers_requested_total_to_connect[$rate_interval])", "interval": "", "legendFormat": "to connect", + "range": true, "refId": "A" }, { @@ -2014,17 +1849,19 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "increase(lodestar_peers_requested_total_to_disconnect[$rate_interval])", + "expr": "60*rate(lodestar_peers_requested_total_to_disconnect[$rate_interval])", "hide": false, "interval": "", "legendFormat": "to disconnect {{reason}}", + "range": true, "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Requested peers to connect and disconnect", + "title": "Requested peers to connect and disconnect (rate / min)", "tooltip": { "shared": true, "sort": 0, @@ -2109,16 +1946,18 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "increase(lodestar_peers_requested_total_subnets_to_query[$rate_interval])", + "expr": "60*rate(lodestar_peers_requested_total_subnets_to_query[$rate_interval])", "interval": "", "legendFormat": "{{type}}", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Requested subnets to query", + "title": "Requested subnets to query (rate / min)", "tooltip": { "shared": true, "sort": 0, @@ -2203,16 +2042,18 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "increase(lodestar_peers_requested_total_subnets_peers_count[$rate_interval])", + "expr": "60*rate(lodestar_peers_requested_total_subnets_peers_count[$rate_interval])", "interval": "", "legendFormat": "{{type}}", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Requested total peers in subnets", + "title": "Requested total peers in subnets (rate / min)", "tooltip": { "shared": true, "sort": 0, @@ -2269,278 +2110,254 @@ "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], "unit": "percentunit" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 79 }, - "hiddenSeries": false, "id": 77, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, "pluginVersion": "9.3.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(lodestar_gossip_validation_queue_job_time_seconds_sum[$rate_interval])", + "editorMode": "code", + "expr": "rate(lodestar_gossip_validation_queue_job_time_seconds_sum[$rate_interval])\n> 0.1/100", "interval": "", "legendFormat": "{{topic}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Gossip validation queue - Utilization rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], "unit": "none" }, "overrides": [] }, - "fill": 1, - "fillGradient": 1, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 79 }, - "hiddenSeries": false, "id": 80, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "expr": "12*rate(lodestar_gossip_validation_queue_job_time_seconds_count[$rate_interval])", "interval": "", "legendFormat": "{{topic}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Gossip validation queue - Processed jobs / slot", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], "unit": "s" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 0, "y": 87 }, - "hiddenSeries": false, "id": 79, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, "pluginVersion": "9.3.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_gossip_validation_queue_job_time_seconds_sum[$rate_interval])/delta(lodestar_gossip_validation_queue_job_time_seconds_count[$rate_interval])", + "editorMode": "code", + "expr": "rate(lodestar_gossip_validation_queue_job_time_seconds_sum[$rate_interval])\n/\nrate(lodestar_gossip_validation_queue_job_time_seconds_count[$rate_interval])\n> 10/1000", "format": "time_series", "interval": "", "legendFormat": "{{topic}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Gossip validation queues - Job time (async)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "datasource": { @@ -2584,19 +2401,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -2628,7 +2432,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_gossip_validation_queue_dropped_jobs_total[$rate_interval])/(delta(lodestar_gossip_validation_queue_job_time_seconds_count[$rate_interval])+delta(lodestar_gossip_validation_queue_dropped_jobs_total[$rate_interval]))", + "expr": "rate(lodestar_gossip_validation_queue_dropped_jobs_total[$rate_interval])/(rate(lodestar_gossip_validation_queue_job_time_seconds_count[$rate_interval])+rate(lodestar_gossip_validation_queue_dropped_jobs_total[$rate_interval]))", "interval": "", "legendFormat": "{{topic}}", "refId": "A" @@ -2638,187 +2442,171 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "unit": "s" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none" }, "overrides": [] }, - "fill": 1, - "fillGradient": 1, "gridPos": { - "h": 8, + "h": 10, "w": 12, - "x": 0, + "x": 12, "y": 95 }, - "hiddenSeries": false, - "id": 82, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 88, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, "pluginVersion": "9.3.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "delta(lodestar_gossip_validation_queue_job_wait_time_seconds_sum[$rate_interval])/delta(lodestar_gossip_validation_queue_job_wait_time_seconds_count[$rate_interval])", - "format": "time_series", + "editorMode": "code", + "expr": "avg_over_time(lodestar_gossip_validation_queue_length[$rate_interval]) > 1", "interval": "", "legendFormat": "{{topic}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "Gossip validation queues - Job wait time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "title": "Gossip validation queues - Length", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "unit": "none" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" }, "overrides": [] }, - "fill": 1, - "fillGradient": 1, "gridPos": { - "h": 8, + "h": 10, "w": 12, - "x": 12, - "y": 95 - }, - "hiddenSeries": false, - "id": 88, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "x": 0, + "y": 96 }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", + "id": 82, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, "pluginVersion": "9.3.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "lodestar_gossip_validation_queue_length", + "editorMode": "code", + "expr": "rate(lodestar_gossip_validation_queue_job_wait_time_seconds_sum[$rate_interval])\n/\nrate(lodestar_gossip_validation_queue_job_wait_time_seconds_count[$rate_interval])\n> 1.2/1000", + "format": "time_series", "interval": "", "legendFormat": "{{topic}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "Gossip validation queues - Length", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "title": "Gossip validation queues - Job wait time", + "type": "timeseries" }, { "datasource": { @@ -2849,8 +2637,7 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 2, - "type": "log" + "type": "linear" }, "showPoints": "auto", "spanNulls": false, @@ -2863,19 +2650,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -2883,10 +2657,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 103 + "x": 12, + "y": 105 }, - "id": 244, + "id": 333, "options": { "legend": { "calcs": [], @@ -2898,45 +2672,35 @@ "mode": "multi", "sort": "none" } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "delta(lodestar_gossip_block_elapsed_time_till_received_sum[$rate_interval])\n/\ndelta(lodestar_gossip_block_elapsed_time_till_received_count[$rate_interval])", - "interval": "", - "legendFormat": "till received", - "refId": "A" - }, + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": false, - "expr": "delta(lodestar_gossip_block_elapsed_time_till_processed_sum[$rate_interval])\n/\ndelta(lodestar_gossip_block_elapsed_time_till_processed_count[$rate_interval])", - "hide": false, + "expr": "(\n rate(lodestar_gossip_block_elapsed_time_till_processed_sum[$rate_interval])\n -\n rate(lodestar_gossip_block_elapsed_time_till_received_sum[$rate_interval])\n)\n/\nrate(lodestar_gossip_block_elapsed_time_till_received_count[$rate_interval])", "interval": "", - "legendFormat": "till processed", - "refId": "B" + "legendFormat": "processed minus received", + "range": true, + "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "delta(lodestar_gossip_block_elapsed_time_till_become_head_sum[$rate_interval])\n/\ndelta(lodestar_gossip_block_elapsed_time_till_become_head_count[$rate_interval])", + "editorMode": "code", + "expr": "rate(lodestar_execution_engine_http_client_request_time_seconds_sum{routeId=\"notifyNewPayload\"} [$rate_interval])\n/\nrate(lodestar_execution_engine_http_client_request_time_seconds_count{routeId=\"notifyNewPayload\"} [$rate_interval])", "hide": false, - "interval": "", - "legendFormat": "till become head", - "refId": "C" + "legendFormat": "notifyNewPayload roundtrip", + "range": true, + "refId": "B" } ], - "title": "Gossip Block Received Delay", + "title": "Gossip Block process time", "type": "timeseries" }, { @@ -2968,7 +2732,8 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "type": "linear" + "log": 2, + "type": "log" }, "showPoints": "auto", "spanNulls": false, @@ -2981,19 +2746,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -3001,10 +2753,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 103 + "x": 0, + "y": 106 }, - "id": 333, + "id": 244, "options": { "legend": { "calcs": [], @@ -3023,12 +2775,10 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", "exemplar": false, - "expr": "(\n rate(lodestar_gossip_block_elapsed_time_till_processed_sum[$rate_interval])\n -\n rate(lodestar_gossip_block_elapsed_time_till_received_sum[$rate_interval])\n)\n/\nrate(lodestar_gossip_block_elapsed_time_till_received_count[$rate_interval])", + "expr": "rate(lodestar_gossip_block_elapsed_time_till_received_sum[$rate_interval])\n/\nrate(lodestar_gossip_block_elapsed_time_till_received_count[$rate_interval])", "interval": "", - "legendFormat": "processed minus received", - "range": true, + "legendFormat": "till received", "refId": "A" }, { @@ -3036,15 +2786,27 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "rate(lodestar_execution_engine_http_client_request_time_seconds_sum{routeId=\"notifyNewPayload\"} [$rate_interval])\n/\nrate(lodestar_execution_engine_http_client_request_time_seconds_count{routeId=\"notifyNewPayload\"} [$rate_interval])", + "exemplar": false, + "expr": "rate(lodestar_gossip_block_elapsed_time_till_processed_sum[$rate_interval])\n/\nrate(lodestar_gossip_block_elapsed_time_till_processed_count[$rate_interval])", "hide": false, - "legendFormat": "notifyNewPayload roundtrip", - "range": true, + "interval": "", + "legendFormat": "till processed", "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rate(lodestar_gossip_block_elapsed_time_till_become_head_sum[$rate_interval])\n/\nrate(lodestar_gossip_block_elapsed_time_till_become_head_count[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "till become head", + "refId": "C" } ], - "title": "Gossip Block process time", + "title": "Gossip Block Received Delay", "type": "timeseries" }, { @@ -3053,7 +2815,7 @@ "h": 1, "w": 24, "x": 0, - "y": 111 + "y": 114 }, "id": 524, "panels": [], @@ -3100,20 +2862,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -3146,7 +2895,7 @@ "h": 8, "w": 12, "x": 0, - "y": 112 + "y": 115 }, "id": 514, "options": { @@ -3242,19 +2991,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -3263,7 +2999,7 @@ "h": 5, "w": 12, "x": 12, - "y": 112 + "y": 115 }, "id": 520, "options": { @@ -3336,19 +3072,6 @@ }, "mappings": [], "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -3357,7 +3080,7 @@ "h": 5, "w": 12, "x": 12, - "y": 117 + "y": 120 }, "id": 522, "options": { @@ -3413,7 +3136,7 @@ "h": 8, "w": 12, "x": 0, - "y": 120 + "y": 123 }, "id": 518, "options": { @@ -3425,8 +3148,8 @@ "mode": "scheme", "reverse": false, "scale": "exponential", - "scheme": "Oranges", - "steps": 64 + "scheme": "Magma", + "steps": 24 }, "exemplars": { "color": "rgba(255,0,255,0.7)" @@ -3459,6 +3182,7 @@ "editorMode": "code", "expr": "rate(lodestar_network_processor_execute_jobs_submitted_total_bucket[$rate_interval])", "format": "heatmap", + "interval": "", "legendFormat": "{{le}}", "range": true, "refId": "A" @@ -3509,19 +3233,6 @@ }, "mappings": [], "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -3530,7 +3241,7 @@ "h": 6, "w": 12, "x": 12, - "y": 122 + "y": 125 }, "id": 521, "options": { @@ -3603,19 +3314,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -3624,7 +3322,7 @@ "h": 8, "w": 12, "x": 0, - "y": 128 + "y": 131 }, "id": 540, "options": { @@ -3695,20 +3393,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3716,7 +3401,7 @@ "h": 8, "w": 12, "x": 12, - "y": 128 + "y": 131 }, "id": 542, "options": { @@ -3753,7 +3438,7 @@ "h": 1, "w": 24, "x": 0, - "y": 136 + "y": 139 }, "id": 528, "panels": [], @@ -3800,20 +3485,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -3846,7 +3518,7 @@ "h": 8, "w": 12, "x": 0, - "y": 137 + "y": 140 }, "id": 526, "options": { @@ -3918,19 +3590,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [ @@ -3956,7 +3615,7 @@ "h": 8, "w": 12, "x": 12, - "y": 137 + "y": 140 }, "id": 530, "options": { @@ -4039,20 +3698,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -4073,7 +3719,7 @@ "h": 8, "w": 12, "x": 0, - "y": 145 + "y": 148 }, "id": 534, "options": { @@ -4156,20 +3802,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -4177,7 +3810,7 @@ "h": 8, "w": 12, "x": 12, - "y": 145 + "y": 148 }, "id": 532, "options": { @@ -4248,20 +3881,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -4286,7 +3906,7 @@ "h": 8, "w": 12, "x": 0, - "y": 153 + "y": 156 }, "id": 536, "options": { @@ -4369,20 +3989,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -4407,7 +4014,7 @@ "h": 8, "w": 12, "x": 12, - "y": 153 + "y": 156 }, "id": 538, "options": { @@ -4456,7 +4063,7 @@ "h": 1, "w": 24, "x": 0, - "y": 161 + "y": 164 }, "id": 600, "panels": [], @@ -4504,19 +4111,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -4525,7 +4119,7 @@ "h": 8, "w": 12, "x": 0, - "y": 162 + "y": 165 }, "id": 602, "options": { @@ -4645,19 +4239,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -4666,7 +4247,7 @@ "h": 8, "w": 12, "x": 12, - "y": 162 + "y": 165 }, "id": 604, "options": { @@ -4785,20 +4366,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -4831,7 +4399,7 @@ "h": 8, "w": 12, "x": 0, - "y": 170 + "y": 173 }, "id": 603, "options": { @@ -4902,53 +4470,15 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "GOSSIP_ERROR_PAST_SLOT" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 170 + "y": 173 }, "id": 601, "options": { @@ -4989,7 +4519,7 @@ "h": 1, "w": 24, "x": 0, - "y": 178 + "y": 181 }, "id": 188, "panels": [], @@ -5046,20 +4576,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -5067,7 +4584,7 @@ "h": 8, "w": 12, "x": 0, - "y": 179 + "y": 182 }, "id": 180, "options": { @@ -5140,19 +4657,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -5161,7 +4665,7 @@ "h": 8, "w": 12, "x": 12, - "y": 179 + "y": 182 }, "id": 176, "options": { @@ -5234,19 +4738,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -5255,7 +4746,7 @@ "h": 8, "w": 12, "x": 0, - "y": 187 + "y": 190 }, "id": 182, "options": { @@ -5277,7 +4768,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(beacon_reqresp_incoming_requests_error_total[$rate_interval])/(delta(beacon_reqresp_incoming_requests_total[$rate_interval])>0)", + "expr": "rate(beacon_reqresp_incoming_requests_error_total[$rate_interval])/(rate(beacon_reqresp_incoming_requests_total[$rate_interval])>0)", "interval": "", "legendFormat": "{{method}}", "refId": "A" @@ -5328,19 +4819,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -5349,7 +4827,7 @@ "h": 8, "w": 12, "x": 12, - "y": 187 + "y": 190 }, "id": 178, "options": { @@ -5371,7 +4849,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(beacon_reqresp_outgoing_requests_error_total[$rate_interval])/(delta(beacon_reqresp_outgoing_requests_total[$rate_interval]) > 0)", + "expr": "rate(beacon_reqresp_outgoing_requests_error_total[$rate_interval])/(rate(beacon_reqresp_outgoing_requests_total[$rate_interval]) > 0)", "interval": "", "legendFormat": "{{method}}", "refId": "A" @@ -5422,19 +4900,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -5443,7 +4908,7 @@ "h": 8, "w": 12, "x": 0, - "y": 195 + "y": 198 }, "id": 605, "options": { @@ -5518,19 +4983,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -5539,7 +4991,7 @@ "h": 8, "w": 12, "x": 12, - "y": 195 + "y": 198 }, "id": 606, "options": { @@ -5614,19 +5066,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -5635,7 +5074,7 @@ "h": 8, "w": 12, "x": 0, - "y": 203 + "y": 206 }, "id": 498, "options": { @@ -5708,19 +5147,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -5729,7 +5155,7 @@ "h": 8, "w": 12, "x": 12, - "y": 203 + "y": 206 }, "id": 500, "options": { @@ -5802,19 +5228,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -5823,7 +5236,7 @@ "h": 8, "w": 12, "x": 0, - "y": 211 + "y": 214 }, "id": 184, "options": { @@ -5896,19 +5309,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -5917,7 +5317,7 @@ "h": 8, "w": 12, "x": 12, - "y": 211 + "y": 214 }, "id": 501, "options": { diff --git a/dashboards/lodestar_rest_api.json b/dashboards/lodestar_rest_api.json index 477071f1ccd4..cadd029cd69e 100644 --- a/dashboards/lodestar_rest_api.json +++ b/dashboards/lodestar_rest_api.json @@ -99,19 +99,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -188,19 +175,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -275,19 +249,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "Bps" }, "overrides": [] @@ -372,20 +333,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -457,20 +405,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -542,20 +477,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, diff --git a/dashboards/lodestar_state_cache_regen.json b/dashboards/lodestar_state_cache_regen.json index 465d0e2eff8e..0d1360837d7f 100644 --- a/dashboards/lodestar_state_cache_regen.json +++ b/dashboards/lodestar_state_cache_regen.json @@ -99,20 +99,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -213,20 +200,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -327,20 +301,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -408,20 +369,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -497,20 +445,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -582,20 +517,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -668,20 +590,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -824,20 +733,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -981,19 +877,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [ @@ -1138,19 +1021,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [ @@ -1308,20 +1178,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1393,20 +1250,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1479,19 +1323,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -1521,7 +1352,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum by (entrypoint) (\n delta(lodestar_regen_fn_call_duration_sum[$rate_interval])\n)\n/\nsum by (entrypoint) (\n delta(lodestar_regen_fn_call_duration_count[$rate_interval])\n)", + "expr": "sum by (entrypoint) (\n rate(lodestar_regen_fn_call_duration_sum[$rate_interval])\n)\n/\nsum by (entrypoint) (\n rate(lodestar_regen_fn_call_duration_count[$rate_interval])\n)", "interval": "", "legendFormat": "{{entrypoint}}", "refId": "A" @@ -1565,19 +1396,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -1607,7 +1425,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum by (caller) (\n delta(lodestar_regen_fn_call_duration_sum[$rate_interval])\n)\n/\nsum by (caller) (\n delta(lodestar_regen_fn_call_duration_count[$rate_interval])\n)", + "expr": "sum by (caller) (\n rate(lodestar_regen_fn_call_duration_sum[$rate_interval])\n)\n/\nsum by (caller) (\n rate(lodestar_regen_fn_call_duration_count[$rate_interval])\n)", "interval": "", "legendFormat": "{{caller}}", "refId": "A" @@ -1653,19 +1471,6 @@ "mappings": [], "max": 1, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1695,7 +1500,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum by (entrypoint) (\n delta(lodestar_regen_fn_queued_total[$rate_interval])\n)\n/\nsum by (entrypoint) (\n delta(lodestar_regen_fn_call_total[$rate_interval])\n)", + "expr": "sum by (entrypoint) (\n rate(lodestar_regen_fn_queued_total[$rate_interval])\n)\n/\nsum by (entrypoint) (\n rate(lodestar_regen_fn_call_total[$rate_interval])\n)", "interval": "", "legendFormat": "", "refId": "A" @@ -1741,19 +1546,6 @@ "mappings": [], "max": 1, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1829,19 +1621,6 @@ "mappings": [], "max": 1, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1871,7 +1650,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum by (caller) (\n delta(lodestar_regen_fn_queued_total[$rate_interval])\n)\n/\nsum by (caller) (\n delta(lodestar_regen_fn_call_total[$rate_interval])\n)", + "expr": "sum by (caller) (\n rate(lodestar_regen_fn_queued_total[$rate_interval])\n)\n/\nsum by (caller) (\n rate(lodestar_regen_fn_call_total[$rate_interval])\n)", "interval": "", "legendFormat": "", "refId": "A" @@ -1917,19 +1696,6 @@ "mappings": [], "max": 1, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -2017,19 +1783,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -2106,19 +1859,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -2190,19 +1930,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -2229,7 +1956,7 @@ "pluginVersion": "7.4.5", "targets": [ { - "expr": "delta(lodestar_regen_queue_dropped_jobs_total[$rate_interval])/(delta(lodestar_regen_queue_job_time_seconds_count[$rate_interval])+delta(lodestar_regen_queue_dropped_jobs_total[$rate_interval]))", + "expr": "rate(lodestar_regen_queue_dropped_jobs_total[$rate_interval])/(rate(lodestar_regen_queue_job_time_seconds_count[$rate_interval])+rate(lodestar_regen_queue_dropped_jobs_total[$rate_interval]))", "interval": "", "legendFormat": "regen_queue", "refId": "A" @@ -2274,19 +2001,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -2313,7 +2027,7 @@ "pluginVersion": "7.4.5", "targets": [ { - "expr": "delta(lodestar_regen_queue_job_time_seconds_sum[$rate_interval])/delta(lodestar_regen_queue_job_time_seconds_count[$rate_interval])", + "expr": "rate(lodestar_regen_queue_job_time_seconds_sum[$rate_interval])/rate(lodestar_regen_queue_job_time_seconds_count[$rate_interval])", "interval": "", "legendFormat": "regen_queue", "refId": "A" @@ -2358,19 +2072,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -2397,7 +2098,7 @@ "pluginVersion": "7.4.5", "targets": [ { - "expr": "delta(lodestar_regen_queue_job_wait_time_seconds_sum[$rate_interval])/delta(lodestar_regen_queue_job_wait_time_seconds_count[$rate_interval])", + "expr": "rate(lodestar_regen_queue_job_wait_time_seconds_sum[$rate_interval])/rate(lodestar_regen_queue_job_wait_time_seconds_count[$rate_interval])", "interval": "", "legendFormat": "regen_queue", "refId": "A" @@ -2442,19 +2143,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] diff --git a/dashboards/lodestar_summary.json b/dashboards/lodestar_summary.json index 349fab48a81a..f5fcedd9f032 100644 --- a/dashboards/lodestar_summary.json +++ b/dashboards/lodestar_summary.json @@ -135,19 +135,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "locale" }, "overrides": [ @@ -340,16 +327,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -399,16 +377,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -457,16 +426,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -515,16 +475,7 @@ }, "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -577,15 +528,6 @@ "mappings": [], "max": 1, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -637,15 +579,6 @@ "fieldConfig": { "defaults": { "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "decbytes" }, "overrides": [] @@ -697,19 +630,6 @@ "fieldConfig": { "defaults": { "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "dateTimeFromNow" }, "overrides": [] @@ -780,15 +700,6 @@ "type": "value" } ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "none" }, "overrides": [] @@ -841,20 +752,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -910,11 +808,7 @@ "fixedColor": "green", "mode": "fixed" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - } + "mappings": [] }, "overrides": [] }, @@ -970,20 +864,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1068,19 +949,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1175,19 +1043,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1295,19 +1150,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1402,19 +1244,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -1446,7 +1275,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "avg(\n delta(validator_monitor_prev_epoch_on_chain_balance[32m])\n)", + "expr": "avg(\n rate(validator_monitor_prev_epoch_on_chain_balance[32m])\n)", "hide": false, "interval": "", "legendFormat": "balance_delta", @@ -1467,19 +1296,6 @@ "mode": "thresholds" }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1565,19 +1381,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1730,19 +1533,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1822,20 +1612,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1915,19 +1692,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -2051,20 +1815,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2116,20 +1867,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2181,20 +1919,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2246,20 +1971,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2311,20 +2023,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2389,20 +2088,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2484,20 +2170,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2527,7 +2200,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(beacon_fork_choice_reorg_total[$rate_interval])", + "expr": "rate(beacon_fork_choice_reorg_total[$rate_interval])", "interval": "", "legendFormat": "", "refId": "A" @@ -2796,19 +2469,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2902,18 +2563,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -3023,18 +2672,6 @@ ], "max": 3, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -3120,18 +2757,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] diff --git a/dashboards/lodestar_sync.json b/dashboards/lodestar_sync.json index 58c9318ccaac..d75b8b4a74a8 100644 --- a/dashboards/lodestar_sync.json +++ b/dashboards/lodestar_sync.json @@ -126,20 +126,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -258,19 +245,6 @@ ], "max": 3, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -354,20 +328,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -446,20 +407,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -538,20 +486,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -630,20 +565,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -722,20 +644,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -852,20 +761,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -971,19 +867,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1125,20 +1008,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -1260,20 +1130,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1364,20 +1221,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1510,19 +1354,6 @@ ], "max": 3, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -1602,20 +1433,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -1727,20 +1545,7 @@ } }, "mappings": [], - "max": 64, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "max": 64 }, "overrides": [] }, @@ -1819,20 +1624,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, diff --git a/dashboards/lodestar_validator_client.json b/dashboards/lodestar_validator_client.json index 2ffbd923d59c..35f7f3ccc458 100644 --- a/dashboards/lodestar_validator_client.json +++ b/dashboards/lodestar_validator_client.json @@ -101,20 +101,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -144,7 +131,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "12 * rate(vc_published_sync_committee_message_total[$__rate_interval])", + "expr": "12 * rate(vc_published_sync_committee_message_total[$rate_interval])", "interval": "", "legendFormat": "sync_committee_message", "refId": "A" @@ -200,11 +187,7 @@ "fixedColor": "green", "mode": "fixed" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - } + "mappings": [] }, "overrides": [] }, @@ -260,20 +243,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -327,20 +297,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -395,20 +352,7 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -461,19 +405,6 @@ "mode": "thresholds" }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "decbytes" }, "overrides": [] @@ -527,19 +458,6 @@ "mode": "thresholds" }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "decbytes" }, "overrides": [] @@ -656,19 +574,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -833,19 +738,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -876,7 +768,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "deriv(vc_proposer_step_call_produce_block_seconds_sum[$__rate_interval])\n/\nrate(vc_proposer_step_call_produce_block_seconds_count[$__rate_interval])", + "expr": "deriv(vc_proposer_step_call_produce_block_seconds_sum[$rate_interval])\n/\nrate(vc_proposer_step_call_produce_block_seconds_count[$rate_interval])", "hide": false, "interval": "", "legendFormat": "call_produce_block", @@ -888,7 +780,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "deriv(vc_proposer_step_call_publish_block_seconds_sum[$__rate_interval])\n/\nrate(vc_proposer_step_call_publish_block_seconds_count[$__rate_interval])", + "expr": "deriv(vc_proposer_step_call_publish_block_seconds_sum[$rate_interval])\n/\nrate(vc_proposer_step_call_publish_block_seconds_count[$rate_interval])", "hide": false, "interval": "", "legendFormat": "call_publish_block", @@ -987,7 +879,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "1000*rate(vc_attester_step_call_publish_attestation_seconds_bucket[$__rate_interval])", + "expr": "1000*rate(vc_attester_step_call_publish_attestation_seconds_bucket[$rate_interval])", "format": "heatmap", "interval": "", "intervalFactor": 10, @@ -1101,7 +993,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "1000*rate(vc_attester_step_call_publish_aggregate_seconds_bucket[$__rate_interval])", + "expr": "1000*rate(vc_attester_step_call_publish_aggregate_seconds_bucket[$rate_interval])", "format": "heatmap", "interval": "", "intervalFactor": 10, @@ -1214,7 +1106,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(vc_sync_committee_step_call_publish_message_seconds_bucket[$__rate_interval])", + "expr": "rate(vc_sync_committee_step_call_publish_message_seconds_bucket[$rate_interval])", "format": "heatmap", "interval": "", "intervalFactor": 10, @@ -1326,7 +1218,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(vc_proposer_step_call_publish_block_seconds_bucket[$__rate_interval])", + "expr": "rate(vc_proposer_step_call_publish_block_seconds_bucket[$rate_interval])", "format": "heatmap", "interval": "", "intervalFactor": 10, @@ -1391,19 +1283,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -1484,19 +1363,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -1579,20 +1445,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1622,7 +1475,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(vc_block_produced_total[$__rate_interval])", + "expr": "rate(vc_block_produced_total[$rate_interval])", "interval": "", "legendFormat": "block produced", "refId": "A" @@ -1633,7 +1486,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "delta(vc_block_published_total[$__rate_interval])", + "expr": "rate(vc_block_published_total[$rate_interval])", "hide": false, "interval": "", "legendFormat": "block published", @@ -1645,7 +1498,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(vc_block_proposing_errors_total[$__rate_interval])", + "expr": "rate(vc_block_proposing_errors_total[$rate_interval])", "hide": false, "interval": "", "legendFormat": "block proposing errors {{error}}", @@ -1696,19 +1549,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1789,19 +1629,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -1929,7 +1756,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(vc_local_sign_time_seconds_bucket[$__rate_interval])", + "expr": "rate(vc_local_sign_time_seconds_bucket[$rate_interval])", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", @@ -1992,20 +1819,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [ { @@ -2134,20 +1948,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -2177,7 +1978,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(vc_proposer_duties_reorg_total[$__rate_interval])", + "expr": "rate(vc_proposer_duties_reorg_total[$rate_interval])", "interval": "", "legendFormat": "proposer duties", "refId": "A" @@ -2188,7 +1989,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(vc_attestation_duties_reorg_total[$__rate_interval])", + "expr": "rate(vc_attestation_duties_reorg_total[$rate_interval])", "hide": false, "interval": "", "legendFormat": "attestation duties", @@ -2200,7 +2001,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(vc_sync_committee_duties_reorg_total[$__rate_interval])", + "expr": "rate(vc_sync_committee_duties_reorg_total[$rate_interval])", "hide": false, "interval": "", "legendFormat": "sync committee duties", diff --git a/dashboards/lodestar_validator_monitor.json b/dashboards/lodestar_validator_monitor.json index 6843cd420b0d..00d18c97bf02 100644 --- a/dashboards/lodestar_validator_monitor.json +++ b/dashboards/lodestar_validator_monitor.json @@ -1,12 +1,12 @@ { "__inputs": [ { - "description": "", - "label": "Prometheus", "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "description": "", "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" + "pluginName": "Prometheus" } ], "annotations": { @@ -63,26 +63,13 @@ "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 4, + "h": 5, + "w": 5, "x": 0, "y": 0 }, @@ -128,47 +115,32 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false + "mode": "continuous-GrYlRd" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, - "x": 4, + "h": 9, + "w": 7, + "x": 5, "y": 0 }, - "id": 4, + "id": 30, "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "show": false + "fields": "", + "values": false }, - "frameIndex": 0, - "showHeader": true + "showUnfilled": true }, "pluginVersion": "9.3.2", "targets": [ @@ -177,44 +149,16 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "1 - validator_monitor_prev_epoch_on_chain_attester_correct_head_total", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "{{index}}", + "editorMode": "code", + "expr": "384*rate(validator_monitor_prev_epoch_attestation_summary{summary!=\"timely_head\"} [$rate_interval])", + "legendFormat": "{{summary}}", + "range": true, "refId": "A" } ], - "title": "Connected validators", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true, - "client_name": true, - "group": false, - "instance": true, - "job": true, - "network": false - }, - "indexByName": { - "Time": 1, - "Value": 8, - "client_name": 2, - "group": 3, - "index": 0, - "instance": 4, - "job": 5, - "network": 6 - }, - "renameByName": {} - } - } - ], - "type": "table" + "title": "Attestation performance summary (rate / epoch)", + "transformations": [], + "type": "bargauge" }, { "datasource": { @@ -233,7 +177,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -246,8 +190,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -256,40 +200,83 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" + "mappings": [] }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 12, "y": 0 }, - "id": 6, + "id": 29, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "8.4.0-beta1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "384*rate(validator_monitor_prev_epoch_attestation_summary{summary!=\"timely_head\"} [$rate_interval])", + "legendFormat": "{{summary}}", + "range": true, + "refId": "A" + } + ], + "title": "Attestation performance summary (rate / epoch)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 5 + }, + "id": 4, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true + }, + "pluginVersion": "9.3.2", "targets": [ { "datasource": { @@ -297,15 +284,43 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "avg(\n delta(validator_monitor_prev_epoch_on_chain_balance[32m])\n)", - "hide": false, + "expr": "1 - validator_monitor_prev_epoch_on_chain_attester_correct_head_total", + "format": "table", + "instant": true, "interval": "", - "legendFormat": "balance_delta", + "legendFormat": "{{index}}", "refId": "A" } ], - "title": "balance delta prev epoch", - "type": "timeseries" + "title": "Connected validators", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "client_name": true, + "group": false, + "instance": true, + "job": true, + "network": false + }, + "indexByName": { + "Time": 1, + "Value": 8, + "client_name": 2, + "group": 3, + "index": 0, + "instance": 4, + "job": 5, + "network": 6 + }, + "renameByName": {} + } + } + ], + "type": "table" }, { "datasource": { @@ -332,7 +347,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 9 }, "id": 27, "options": { @@ -431,20 +446,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "unit": "none" }, "overrides": [] }, @@ -452,9 +454,9 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 9 }, - "id": 10, + "id": 6, "options": { "legend": { "calcs": [], @@ -475,25 +477,14 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "avg(validator_monitor_prev_epoch_on_chain_inclusion_distance)", - "interval": "", - "legendFormat": "inclusion distance", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "rate(validator_monitor_prev_epoch_on_chain_inclusion_distance_sum[$rate_interval])\n/\nrate(validator_monitor_prev_epoch_on_chain_inclusion_distance_count[$rate_interval])", + "expr": "avg(\n rate(validator_monitor_prev_epoch_on_chain_balance[32m])\n)", "hide": false, "interval": "", - "legendFormat": "inclusion distance new", - "refId": "B" + "legendFormat": "balance_delta", + "refId": "A" } ], - "title": "Avg inclusion distance", + "title": "balance delta prev epoch", "type": "timeseries" }, { @@ -538,19 +529,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -559,7 +537,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 17 }, "id": 8, "options": { @@ -595,7 +573,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Min delay from when the validator should send an object and when it was received", "fieldConfig": { "defaults": { "color": { @@ -632,20 +609,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" + "unit": "short" }, "overrides": [] }, @@ -653,15 +617,15 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 17 }, - "id": 18, + "id": 10, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -676,22 +640,10 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum(delta(validator_monitor_prev_epoch_attestations_min_delay_seconds_sum[$rate_interval]))\n/\nsum(delta(validator_monitor_prev_epoch_attestations_min_delay_seconds_count[$rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "attestations", - "refId": "Attestations" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(delta(validator_monitor_prev_epoch_aggregates_min_delay_seconds_sum[$rate_interval]))\n/\nsum(delta(validator_monitor_prev_epoch_aggregates_min_delay_seconds_count[$rate_interval]))", + "expr": "avg(validator_monitor_prev_epoch_on_chain_inclusion_distance)", "interval": "", - "legendFormat": "aggregates", - "refId": "Aggregates" + "legendFormat": "inclusion distance", + "refId": "A" }, { "datasource": { @@ -699,14 +651,14 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum(delta(validator_monitor_prev_epoch_beacon_blocks_min_delay_seconds_sum[$rate_interval]))\n/\nsum(delta(validator_monitor_prev_epoch_beacon_blocks_min_delay_seconds_count[$rate_interval]))", + "expr": "rate(validator_monitor_prev_epoch_on_chain_inclusion_distance_sum[$rate_interval])\n/\nrate(validator_monitor_prev_epoch_on_chain_inclusion_distance_count[$rate_interval])", "hide": false, "interval": "", - "legendFormat": "", - "refId": "Blocks" + "legendFormat": "inclusion distance new", + "refId": "B" } ], - "title": "Prev epoch min delay", + "title": "Avg inclusion distance", "type": "timeseries" }, { @@ -751,19 +703,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -772,7 +711,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 25 }, "id": 12, "options": { @@ -796,7 +735,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum(delta(validator_monitor_prev_epoch_on_chain_attester_miss_total[$rate_interval]))\n/\n(\n sum(delta(validator_monitor_prev_epoch_on_chain_attester_hit_total[$rate_interval]))\n +\n sum(delta(validator_monitor_prev_epoch_on_chain_attester_miss_total[$rate_interval]))\n) > 0", + "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_attester_miss_total[$rate_interval]))\n/\n(\n sum(rate(validator_monitor_prev_epoch_on_chain_attester_hit_total[$rate_interval]))\n +\n sum(rate(validator_monitor_prev_epoch_on_chain_attester_miss_total[$rate_interval]))\n) > 0", "interval": "", "legendFormat": "attester", "refId": "A" @@ -807,7 +746,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum(delta(validator_monitor_prev_epoch_on_chain_target_attester_miss_total[$rate_interval]))\n/\n(\n sum(delta(validator_monitor_prev_epoch_on_chain_target_attester_hit_total[$rate_interval]))\n +\n sum(delta(validator_monitor_prev_epoch_on_chain_target_attester_miss_total[$rate_interval]))\n) > 0", + "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_target_attester_miss_total[$rate_interval]))\n/\n(\n sum(rate(validator_monitor_prev_epoch_on_chain_target_attester_hit_total[$rate_interval]))\n +\n sum(rate(validator_monitor_prev_epoch_on_chain_target_attester_miss_total[$rate_interval]))\n) > 0", "hide": false, "interval": "", "legendFormat": "target", @@ -819,7 +758,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum(delta(validator_monitor_prev_epoch_on_chain_head_attester_miss_total[$rate_interval]))\n/\n(\n sum(delta(validator_monitor_prev_epoch_on_chain_head_attester_hit_total[$rate_interval]))\n +\n sum(delta(validator_monitor_prev_epoch_on_chain_head_attester_miss_total[$rate_interval]))\n) > 0", + "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_head_attester_miss_total[$rate_interval]))\n/\n(\n sum(rate(validator_monitor_prev_epoch_on_chain_head_attester_hit_total[$rate_interval]))\n +\n sum(rate(validator_monitor_prev_epoch_on_chain_head_attester_miss_total[$rate_interval]))\n) > 0", "hide": false, "interval": "", "legendFormat": "head", @@ -834,6 +773,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Min delay from when the validator should send an object and when it was received", "fieldConfig": { "defaults": { "color": { @@ -863,27 +803,14 @@ "spanNulls": true, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" + "unit": "s" }, "overrides": [] }, @@ -891,15 +818,15 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 25 }, - "id": 14, + "id": 18, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "mode": "multi", @@ -914,11 +841,11 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "count(validator_monitor_prev_epoch_on_chain_inclusion_distance == 1) / count(validator_monitor_prev_epoch_on_chain_inclusion_distance)", + "expr": "sum(rate(validator_monitor_prev_epoch_attestations_min_delay_seconds_sum[$rate_interval]))\n/\nsum(rate(validator_monitor_prev_epoch_attestations_min_delay_seconds_count[$rate_interval]))", "hide": false, "interval": "", - "legendFormat": "1", - "refId": "D" + "legendFormat": "attestations", + "refId": "Attestations" }, { "datasource": { @@ -926,23 +853,10 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "count(validator_monitor_prev_epoch_on_chain_inclusion_distance == 2) / count(validator_monitor_prev_epoch_on_chain_inclusion_distance)", - "hide": false, + "expr": "sum(rate(validator_monitor_prev_epoch_aggregates_min_delay_seconds_sum[$rate_interval]))\n/\nsum(rate(validator_monitor_prev_epoch_aggregates_min_delay_seconds_count[$rate_interval]))", "interval": "", - "legendFormat": "2", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "count(5 > validator_monitor_prev_epoch_on_chain_inclusion_distance >= 3) / count(validator_monitor_prev_epoch_on_chain_inclusion_distance)", - "hide": false, - "interval": "", - "legendFormat": "3-5", - "refId": "E" + "legendFormat": "aggregates", + "refId": "Aggregates" }, { "datasource": { @@ -950,25 +864,14 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "count(10 > validator_monitor_prev_epoch_on_chain_inclusion_distance >= 5) / count(validator_monitor_prev_epoch_on_chain_inclusion_distance)", + "expr": "sum(rate(validator_monitor_prev_epoch_beacon_blocks_min_delay_seconds_sum[$rate_interval]))\n/\nsum(rate(validator_monitor_prev_epoch_beacon_blocks_min_delay_seconds_count[$rate_interval]))", "hide": false, "interval": "", - "legendFormat": "5-10", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "count(validator_monitor_prev_epoch_on_chain_inclusion_distance >= 10) / count(validator_monitor_prev_epoch_on_chain_inclusion_distance)", - "interval": "", - "legendFormat": "+10", - "refId": "A" + "legendFormat": "", + "refId": "Blocks" } ], - "title": "Inclusion distance distribution", + "title": "Prev epoch min delay", "type": "timeseries" }, { @@ -1004,7 +907,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 33 }, "heatmap": {}, "hideZeroBuckets": false, @@ -1101,8 +1004,8 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 8, - "gradientMode": "opacity", + "fillOpacity": 10, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, @@ -1112,34 +1015,20 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 2, - "type": "log" + "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "unit": "percentunit" }, "overrides": [] }, @@ -1147,15 +1036,15 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 33 }, - "id": 20, + "id": 14, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -1170,10 +1059,11 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "validator_monitor_prev_epoch_attestations_count / validator_monitor_validators", + "expr": "count(validator_monitor_prev_epoch_on_chain_inclusion_distance == 1) / count(validator_monitor_prev_epoch_on_chain_inclusion_distance)", + "hide": false, "interval": "", - "legendFormat": "attestations_sent", - "refId": "A" + "legendFormat": "1", + "refId": "D" }, { "datasource": { @@ -1181,14 +1071,49 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "validator_monitor_prev_epoch_aggregates_count / validator_monitor_validators", + "expr": "count(validator_monitor_prev_epoch_on_chain_inclusion_distance == 2) / count(validator_monitor_prev_epoch_on_chain_inclusion_distance)", "hide": false, "interval": "", - "legendFormat": "aggregates_sent", - "refId": "D" + "legendFormat": "2", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "count(5 > validator_monitor_prev_epoch_on_chain_inclusion_distance >= 3) / count(validator_monitor_prev_epoch_on_chain_inclusion_distance)", + "hide": false, + "interval": "", + "legendFormat": "3-5", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "count(10 > validator_monitor_prev_epoch_on_chain_inclusion_distance >= 5) / count(validator_monitor_prev_epoch_on_chain_inclusion_distance)", + "hide": false, + "interval": "", + "legendFormat": "5-10", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "count(validator_monitor_prev_epoch_on_chain_inclusion_distance >= 10) / count(validator_monitor_prev_epoch_on_chain_inclusion_distance)", + "interval": "", + "legendFormat": "+10", + "refId": "A" } ], - "title": "Attestater sent per epoch per validator", + "title": "Inclusion distance distribution", "type": "timeseries" }, { @@ -1224,7 +1149,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 41 }, "heatmap": {}, "hideZeroBuckets": false, @@ -1317,7 +1242,7 @@ "custom": { "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "Gwei", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -1346,19 +1271,199 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.0-beta1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "validator_monitor_prev_epoch_attestations_count / validator_monitor_validators", + "interval": "", + "legendFormat": "attestations_sent", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(validator_monitor_prev_epoch_aggregates_count[$rate_interval]) / validator_monitor_validators", + "hide": false, + "interval": "", + "legendFormat": "aggregates_sent", + "range": true, + "refId": "D" + } + ], + "title": "Attestater sent per epoch per validator", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.0-beta1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "validator_monitor_prev_epoch_attestation_block_inclusions_sum\n/\nvalidator_monitor_prev_epoch_attestation_block_inclusions_count", + "hide": false, + "interval": "", + "legendFormat": "block_inclusions", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "validator_monitor_prev_epoch_attestation_aggregate_inclusions_sum\n/\nvalidator_monitor_prev_epoch_attestation_aggregate_inclusions_count", + "hide": false, + "interval": "", + "legendFormat": "aggregate_inclusions", + "refId": "B" + } + ], + "title": "Attestation inclusions epoch per validator", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Gwei", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], "unit": "short" }, "overrides": [ @@ -1428,7 +1533,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 49 }, "id": 22, "options": { @@ -1513,8 +1618,8 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 8, - "gradientMode": "opacity", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, @@ -1524,11 +1629,10 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 2, - "type": "log" + "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -1538,20 +1642,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "unit": "percentunit" }, "overrides": [] }, @@ -1559,9 +1650,9 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 57 }, - "id": 16, + "id": 32, "options": { "legend": { "calcs": [], @@ -1570,38 +1661,26 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, - "pluginVersion": "8.4.0-beta1", + "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "validator_monitor_prev_epoch_attestation_block_inclusions_sum\n/\nvalidator_monitor_prev_epoch_attestation_block_inclusions_count", - "hide": false, - "interval": "", - "legendFormat": "block_inclusions", + "editorMode": "code", + "expr": "rate(validator_monitor_unaggregated_attestation_submitted_sent_peers_count_bucket{le=\"0\"} [$rate_interval])\n/ on(instance)\nrate(validator_monitor_unaggregated_attestation_submitted_sent_peers_count_count [$rate_interval])", + "format": "time_series", + "legendFormat": "__auto", + "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "validator_monitor_prev_epoch_attestation_aggregate_inclusions_sum\n/\nvalidator_monitor_prev_epoch_attestation_aggregate_inclusions_count", - "hide": false, - "interval": "", - "legendFormat": "aggregate_inclusions", - "refId": "B" } ], - "title": "Attestation inclusions epoch per validator", + "title": "Unaggregated attestations submitted to zero peers", "type": "timeseries" } ], diff --git a/dashboards/lodestar_vm_host.json b/dashboards/lodestar_vm_host.json index a5a3a579535e..799f32cd2bc3 100644 --- a/dashboards/lodestar_vm_host.json +++ b/dashboards/lodestar_vm_host.json @@ -1,12 +1,19 @@ { "__inputs": [ { - "description": "", - "label": "Prometheus", "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "description": "", "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" + "pluginName": "Prometheus" + }, + { + "name": "VAR_BEACON_JOB", + "type": "constant", + "label": "Beacon node job name", + "value": "beacon", + "description": "" } ], "annotations": { @@ -121,19 +128,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "decbytes" }, "overrides": [] @@ -253,15 +247,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -370,19 +355,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -464,15 +436,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "percentunit" }, "overrides": [ @@ -584,19 +547,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "decbytes" }, "overrides": [] @@ -737,19 +687,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [ @@ -847,19 +784,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -969,19 +893,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [ @@ -1030,7 +941,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "nodejs_eventloop_lag_seconds", + "expr": "nodejs_eventloop_lag_seconds{job=~\"$beacon_job|beacon\"}", "interval": "", "legendFormat": "main_thread", "range": true, @@ -1104,20 +1015,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -1199,19 +1097,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [ @@ -1362,19 +1247,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1484,19 +1356,6 @@ ], "max": 3, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -1582,19 +1441,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1745,19 +1591,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1839,19 +1672,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -1932,19 +1752,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "Bps" }, "overrides": [] @@ -2040,19 +1847,6 @@ "links": [], "mappings": [], "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "bytes" }, "overrides": [ @@ -2564,19 +2358,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "Bps" }, "overrides": [] @@ -2669,19 +2450,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [ @@ -2791,19 +2559,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "cps" }, "overrides": [] @@ -2923,19 +2678,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -3047,19 +2789,6 @@ ], "max": 1, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [ @@ -3185,19 +2914,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -3282,19 +2998,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [ @@ -3465,19 +3168,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -3561,19 +3251,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -3655,19 +3332,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "Bps" }, "overrides": [] @@ -3759,20 +3423,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3851,20 +3502,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -3943,20 +3581,7 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "mappings": [] }, "overrides": [] }, @@ -4063,19 +3688,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -4160,19 +3772,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -4257,19 +3856,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -4354,19 +3940,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -4449,19 +4022,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "bytes" }, "overrides": [ @@ -4567,19 +4127,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "bytes" }, "overrides": [ @@ -4686,19 +4233,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] diff --git a/scripts/download_dashboards.mjs b/scripts/download_dashboards.mjs old mode 100644 new mode 100755 diff --git a/scripts/grafana_push_dashboards.sh b/scripts/grafana_push_dashboards.sh index 851193ebc23d..9cc7a7a84a00 100755 --- a/scripts/grafana_push_dashboards.sh +++ b/scripts/grafana_push_dashboards.sh @@ -4,11 +4,11 @@ # # USAGE: # -# API_URL=http://localhost:3000 API_USERPASS=admin:admin ./grafana_push_dashboards.sh dashboards/*.json +# source .secrets.env && scripts/grafana_push_dashboards.sh dashboards/lodestar_*.json # # - Accepts a single file, a file glob, or multiple combinations of both -# - Set API_URL to the root Grafana API url: API_URL=https://yourgrafana.server -# - Set API_USERPASS to `$user:$password`: API_USERPASS=admin:admin +# - Set GRAFANA_URL to the root Grafana API url: GRAFANA_URL=https://yourgrafana.server +# - Set GRAFANA_API_KEY to an authorized token if [ $# -eq 0 ]; then echo "No arguments supplied" @@ -35,9 +35,9 @@ for fileglob in "${@:1}"; do curl -X POST \ -H "Accept: application/json" \ -H "Content-Type: application/json" \ - -u $API_USERPASS \ + -H "Authorization: Bearer $GRAFANA_API_KEY" \ -d @$filename \ - ${API_URL}/api/dashboards/import + ${GRAFANA_URL}/api/dashboards/import done done diff --git a/scripts/lint-grafana-dashboard.mjs b/scripts/lint-grafana-dashboard.mjs index 9cee9b6a5694..ba2b566b1338 100644 --- a/scripts/lint-grafana-dashboard.mjs +++ b/scripts/lint-grafana-dashboard.mjs @@ -10,6 +10,7 @@ import fs from "node:fs"; @typescript-eslint/no-unsafe-assignment, @typescript-eslint/explicit-function-return-type, @typescript-eslint/naming-convention, +quotes, no-console */ @@ -44,9 +45,16 @@ no-console * @property {boolean} [collapsed] * @property {string} title * @property {Datasource} [datasource] + * @property {FieldConfig} [fieldConfig] * @property {Target[]} [targets] * @property {Panel[]} [panels] * + * @typedef {Object} FieldConfig + * @property {FieldConfigDefaults} [defaults] + * + * @typedef {Object} FieldConfigDefaults + * @property {any} [thresholds] + * * @typedef {Object} Target * @property {Datasource} [datasource] * @property {boolean} [exemplar] @@ -317,13 +325,39 @@ function assertPanels(panels) { target.exemplar = false; } - // Force usage of interval variable if (target.expr) { - target.expr.replace(/\$__rate_interval/g, `$${variableNameRateInterval}`); + // Force usage of interval variable + target.expr = target.expr.replace(/\$__rate_interval/g, `$${variableNameRateInterval}`); + + // Ensure to always use variables to match job names + target.expr = target.expr.replace(/job="beacon"/g, 'job=~"$beacon_job|beacon"'); + target.expr = target.expr.replace(/job="validator"/g, 'job=~"$validator_job|validator"'); + + // Ban use of delta and increase functions. + // Mixed use of delta / increase and rate make dashboards more difficult to reason about. + // - delta shows the value difference based on the selected time interval, which is variable + // so if time interval is X or 2*X the displayed values double. + // - rate shows the per-second average rate regardless of time interval. The time interval just + // controls how "smooth" that average is. + // Using rate results in an easier read of values. Sometimes the rate will be scaled up to + // rate / slot, rate / epoch, or rate / min; all of which are clearly indicated in the chart title + if (target.expr.includes("delta(")) { + throw Error(`promql function 'delta' is not allowed, use 'rate' instead: ${target.expr}`); + } + if (target.expr.includes("increase(")) { + throw Error(`promql function 'increase' is not allowed, use 'rate' instead: ${target.expr}`); + } } } } + // Drop threshold defaults at `fieldConfig.defaults.thresholds` + // They are a source of diffs constant since consencutive exports assign null values + // while other don't. Since we do not plan to add default thresholds, drop for now + if (panel.fieldConfig?.defaults?.thresholds) { + delete panel.fieldConfig?.defaults?.thresholds; + } + // Recursively check nested panels if (panel.panels) { assertPanels(panel.panels); From 7fc999e7487aed2f634aecb280e0eccf56458191 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 10 Jul 2023 21:43:03 +0200 Subject: [PATCH 50/96] chore: remove winston from packages other than logger (#5742) --- packages/beacon-node/package.json | 2 +- packages/cli/package.json | 3 --- packages/prover/package.json | 2 -- packages/utils/package.json | 3 +-- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index fbdb5aa9609b..b9b775f72544 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -124,7 +124,6 @@ "@lodestar/db": "^1.9.1", "@lodestar/fork-choice": "^1.9.1", "@lodestar/light-client": "^1.9.1", - "@lodestar/logger": "^1.9.1", "@lodestar/params": "^1.9.1", "@lodestar/reqresp": "^1.9.1", "@lodestar/state-transition": "^1.9.1", @@ -156,6 +155,7 @@ "xxhash-wasm": "1.0.2" }, "devDependencies": { + "@lodestar/logger": "^1.9.1", "@types/eventsource": "^1.1.11", "@types/leveldown": "^4.0.3", "@types/qs": "^6.9.7", diff --git a/packages/cli/package.json b/packages/cli/package.json index e51543e563a3..e2631096887e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -88,9 +88,6 @@ "source-map-support": "^0.5.21", "uint8arrays": "^4.0.3", "uuidv4": "^6.2.13", - "winston": "^3.8.2", - "winston-daily-rotate-file": "^4.7.1", - "winston-transport": "^4.5.0", "yargs": "^17.7.1" }, "devDependencies": { diff --git a/packages/prover/package.json b/packages/prover/package.json index f4da974ff4af..d3122333da49 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -78,8 +78,6 @@ "http-proxy": "^1.18.1", "js-yaml": "^4.1.0", "source-map-support": "^0.5.21", - "winston-transport": "^4.5.0", - "winston": "^3.8.2", "yargs": "^17.7.1" }, "devDependencies": { diff --git a/packages/utils/package.json b/packages/utils/package.json index 270f941d7e6a..00d0bbdfdcd3 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -42,8 +42,7 @@ "bigint-buffer": "^1.1.5", "case": "^1.6.3", "chalk": "^5.2.0", - "js-yaml": "^4.1.0", - "winston": "^3.8.2" + "js-yaml": "^4.1.0" }, "devDependencies": { "@types/js-yaml": "^4.0.5", From 85ff3cf17e9f8338865cc66f129f9e60e0ec029b Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 11 Jul 2023 02:02:22 +0200 Subject: [PATCH 51/96] refactor: increase batch size to get validator indices to 64 (#5743) --- packages/validator/src/services/indices.ts | 4 ++-- packages/validator/src/util/batch.ts | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/validator/src/services/indices.ts b/packages/validator/src/services/indices.ts index 38f2d2a9d1df..a3c29982bf58 100644 --- a/packages/validator/src/services/indices.ts +++ b/packages/validator/src/services/indices.ts @@ -7,9 +7,9 @@ import {Metrics} from "../metrics.js"; /** * URLs have a limitation on size, adding an unbounded num of pubkeys will break the request. - * For reasoning on the specific number see: https://github.com/ChainSafe/lodestar/pull/2730#issuecomment-866749083 + * For reasoning on the specific number see: https://github.com/ethereum/beacon-APIs/pull/328 */ -const PUBKEYS_PER_REQUEST = 10; +const PUBKEYS_PER_REQUEST = 64; // To assist with readability type PubkeyHex = string; diff --git a/packages/validator/src/util/batch.ts b/packages/validator/src/util/batch.ts index f0cfa9c7ee9d..a44d188fbf6b 100644 --- a/packages/validator/src/util/batch.ts +++ b/packages/validator/src/util/batch.ts @@ -1,6 +1,5 @@ /** - * Divide pubkeys into batches, each batch contains at most 5 http requests, - * each request can work on at most 40 pubkeys. + * Convert array of items into array of batched item arrays */ export function batchItems(items: T[], opts: {batchSize: number; maxBatches?: number}): T[][] { const batches: T[][] = []; From 91e2e655da398f6bfeab09880dfdfe127176f121 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 11 Jul 2023 10:27:54 -0400 Subject: [PATCH 52/96] fix: ignore discovered peers with no multiaddrs (#5736) * fix: ignore discovered peers with no multiaddrs * Support undefined multiaddrs --- packages/beacon-node/src/network/peers/discover.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/network/peers/discover.ts b/packages/beacon-node/src/network/peers/discover.ts index 8a5505a6580e..0c64dbdd865e 100644 --- a/packages/beacon-node/src/network/peers/discover.ts +++ b/packages/beacon-node/src/network/peers/discover.ts @@ -51,6 +51,7 @@ enum DiscoveredPeerStatus { attempt_dial = "attempt_dial", cached = "cached", dropped = "dropped", + no_multiaddrs = "no_multiaddrs", } type UnixMs = number; @@ -267,10 +268,18 @@ export class PeerDiscovery { } /** - * Progressively called by libp2p peer discovery as a result of any query. + * Progressively called by libp2p as a result of peer discovery or updates to its peer store */ private onDiscoveredPeer = (evt: CustomEvent): void => { const {id, multiaddrs} = evt.detail; + + // libp2p may send us PeerInfos without multiaddrs https://github.com/libp2p/js-libp2p/issues/1873 + // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions + if (!multiaddrs || multiaddrs.length === 0) { + this.metrics?.discovery.discoveredStatus.inc({status: DiscoveredPeerStatus.no_multiaddrs}); + return; + } + const attnets = zeroAttnets; const syncnets = zeroSyncnets; const status = this.handleDiscoveredPeer(id, multiaddrs[0], attnets, syncnets); From 0640e0613fe8d8b2bdf712e8a54466fb27f3f0d6 Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Tue, 11 Jul 2023 22:51:55 +0700 Subject: [PATCH 53/96] feat: subscribe to 2 long lived subnets per node (#5704) * feat: implement computeSubscribedSubnet() * feat: implement DLLAttnetsService * fix: deterministicLongLivedAttnets feature flag * fix: optional nodeId * chore: add subnet to word list --- .wordlist.txt | 1 + .../beacon-node/src/network/core/metrics.ts | 4 + .../src/network/core/networkCore.ts | 29 +- packages/beacon-node/src/network/options.ts | 2 + .../src/network/peers/utils/subnetMap.ts | 4 - .../src/network/subnets/attnetsService.ts | 8 +- .../src/network/subnets/dllAttnetsService.ts | 315 ++++++++++++++++++ .../src/network/subnets/interface.ts | 9 +- .../beacon-node/src/network/subnets/util.ts | 60 ++++ .../unit-mainnet/network/subnets/util.test.ts | 67 ++++ .../network/subnets/dllAttnetsService.test.ts | 149 +++++++++ .../test/unit/network/subnets/util.test.ts | 41 +++ .../src/options/beaconNodeOptions/network.ts | 9 + .../unit/options/beaconNodeOptions.test.ts | 2 + packages/params/src/index.ts | 4 + 15 files changed, 682 insertions(+), 22 deletions(-) create mode 100644 packages/beacon-node/src/network/subnets/dllAttnetsService.ts create mode 100644 packages/beacon-node/src/network/subnets/util.ts create mode 100644 packages/beacon-node/test/unit-mainnet/network/subnets/util.test.ts create mode 100644 packages/beacon-node/test/unit/network/subnets/dllAttnetsService.test.ts create mode 100644 packages/beacon-node/test/unit/network/subnets/util.test.ts diff --git a/.wordlist.txt b/.wordlist.txt index fbc71353e180..22b2d24cf81e 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -115,6 +115,7 @@ runtime sharding ssz stakers +subnet subnets tcp testnet diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index 95fe62d3557b..a7ff827fd712 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -231,6 +231,10 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_attnets_service_random_subscriptions_total", help: "Count of random subscriptions", }), + longLivedSubscriptions: register.gauge({ + name: "lodestar_attnets_service_long_lived_subscriptions_total", + help: "Count of long lived subscriptions", + }), subscribeSubnets: register.gauge<"subnet" | "src">({ name: "lodestar_attnets_service_subscribe_subnets_total", help: "Count of subscribe_subnets calls", diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts index ee29d37e8efe..19a8c8d0825f 100644 --- a/packages/beacon-node/src/network/core/networkCore.ts +++ b/packages/beacon-node/src/network/core/networkCore.ts @@ -9,6 +9,8 @@ import {LoggerNode} from "@lodestar/logger/node"; import {Epoch, phase0} from "@lodestar/types"; import {ForkName} from "@lodestar/params"; import {ResponseIncoming} from "@lodestar/reqresp"; +import {fromHexString} from "@chainsafe/ssz"; +import {ENR} from "@chainsafe/discv5"; import {Libp2p} from "../interface.js"; import {PeerManager} from "../peers/peerManager.js"; import {ReqRespBeaconNode} from "../reqresp/ReqRespBeaconNode.js"; @@ -18,7 +20,7 @@ import {AttnetsService} from "../subnets/attnetsService.js"; import {SyncnetsService} from "../subnets/syncnetsService.js"; import {FORK_EPOCH_LOOKAHEAD, getActiveForks} from "../forks.js"; import {NetworkOptions} from "../options.js"; -import {CommitteeSubscription} from "../subnets/interface.js"; +import {CommitteeSubscription, IAttnetsService} from "../subnets/interface.js"; import {MetadataController} from "../metadata.js"; import {createNodeJsLibp2p} from "../nodejs/util.js"; import {PeersData} from "../peers/peersData.js"; @@ -31,6 +33,7 @@ import {Discv5Worker} from "../discv5/index.js"; import {LocalStatusCache} from "../statusCache.js"; import {RegistryMetricCreator} from "../../metrics/index.js"; import {peerIdFromString, peerIdToString} from "../../util/peerId.js"; +import {DLLAttnetsService} from "../subnets/dllAttnetsService.js"; import {NetworkCoreMetrics, createNetworkCoreMetrics} from "./metrics.js"; import {INetworkCore, MultiaddrStr, PeerIdStr} from "./types.js"; @@ -38,7 +41,7 @@ type Mods = { libp2p: Libp2p; gossip: Eth2Gossipsub; reqResp: ReqRespBeaconNode; - attnetsService: AttnetsService; + attnetsService: IAttnetsService; syncnetsService: SyncnetsService; peerManager: PeerManager; peersData: PeersData; @@ -84,7 +87,7 @@ export type BaseNetworkInit = { export class NetworkCore implements INetworkCore { // Internal modules private readonly libp2p: Libp2p; - private readonly attnetsService: AttnetsService; + private readonly attnetsService: IAttnetsService; private readonly syncnetsService: SyncnetsService; private readonly peerManager: PeerManager; private readonly peersData: PeersData; @@ -185,7 +188,18 @@ export class NetworkCore implements INetworkCore { events, }); - const attnetsService = new AttnetsService(config, clock, gossip, metadata, logger, metrics, opts); + // Note: should not be necessary, already called in createNodeJsLibp2p() + await libp2p.start(); + + await reqResp.start(); + // should be called before DLLAttnetsService constructor so that node subscribe to deterministic attnet topics + await gossip.start(); + + const enr = opts.discv5?.enr; + const nodeId = enr ? fromHexString(ENR.decodeTxt(enr).nodeId) : null; + const attnetsService = opts.deterministicLongLivedAttnets + ? new DLLAttnetsService(config, clock, gossip, metadata, logger, metrics, nodeId, opts) + : new AttnetsService(config, clock, gossip, metadata, logger, metrics, opts); const syncnetsService = new SyncnetsService(config, clock, gossip, metadata, logger, metrics, opts); const peerManager = await PeerManager.init( @@ -207,13 +221,6 @@ export class NetworkCore implements INetworkCore { opts ); - // Note: should not be necessary, already called in createNodeJsLibp2p() - await libp2p.start(); - - await reqResp.start(); - - await gossip.start(); - // Network spec decides version changes based on clock fork, not head fork const forkCurrentSlot = config.getForkName(clock.currentSlot); // Register only ReqResp protocols relevant to clock's fork diff --git a/packages/beacon-node/src/network/options.ts b/packages/beacon-node/src/network/options.ts index 095d2b0bc3d1..80ffa595622d 100644 --- a/packages/beacon-node/src/network/options.ts +++ b/packages/beacon-node/src/network/options.ts @@ -2,6 +2,7 @@ import {Eth2GossipsubOpts} from "./gossip/gossipsub.js"; import {PeerManagerOpts, PeerRpcScoreOpts} from "./peers/index.js"; import {ReqRespBeaconNodeOpts} from "./reqresp/ReqRespBeaconNode.js"; import {NetworkProcessorOpts} from "./processor/index.js"; +import {SubnetsServiceOpts} from "./subnets/interface.js"; // Since Network is eventually intended to be run in a separate thread, ensure that all options are cloneable using structuredClone export interface NetworkOptions @@ -10,6 +11,7 @@ export interface NetworkOptions Omit, NetworkProcessorOpts, PeerRpcScoreOpts, + SubnetsServiceOpts, Eth2GossipsubOpts { localMultiaddrs: string[]; bootMultiaddrs?: string[]; diff --git a/packages/beacon-node/src/network/peers/utils/subnetMap.ts b/packages/beacon-node/src/network/peers/utils/subnetMap.ts index c69ef1db565c..97985b3d8da0 100644 --- a/packages/beacon-node/src/network/peers/utils/subnetMap.ts +++ b/packages/beacon-node/src/network/peers/utils/subnetMap.ts @@ -44,10 +44,6 @@ export class SubnetMap { return toSlot !== undefined && toSlot >= slot; // ACTIVE: >= } - activeUpToSlot(subnet: number): Slot | null { - return this.subnets.get(subnet) ?? null; - } - /** Return subnetIds with a `toSlot` equal greater than `currentSlot` */ getActive(currentSlot: Slot): number[] { const subnetIds: number[] = []; diff --git a/packages/beacon-node/src/network/subnets/attnetsService.ts b/packages/beacon-node/src/network/subnets/attnetsService.ts index a95834316c38..35f30eb41328 100644 --- a/packages/beacon-node/src/network/subnets/attnetsService.ts +++ b/packages/beacon-node/src/network/subnets/attnetsService.ts @@ -23,6 +23,7 @@ import { RandBetweenFn, ShuffleFn, GossipSubscriber, + SubnetsServiceTestOpts, } from "./interface.js"; /** @@ -78,7 +79,7 @@ export class AttnetsService implements IAttnetsService { private readonly metadata: MetadataController, private readonly logger: Logger, private readonly metrics: NetworkCoreMetrics | null, - private readonly opts?: SubnetsServiceOpts + private readonly opts?: SubnetsServiceOpts & SubnetsServiceTestOpts ) { // if subscribeAllSubnets, we act like we have >= ATTESTATION_SUBNET_COUNT validators connecting to this node // so that we have enough subnet topic peers, see https://github.com/ChainSafe/lodestar/issues/4921 @@ -158,11 +159,6 @@ export class AttnetsService implements IAttnetsService { return this.aggregatorSlotSubnet.getOrDefault(slot).has(subnet); } - /** Returns the latest Slot subscription is active, null if no subscription */ - activeUpToSlot(subnet: number): Slot | null { - return this.subscriptionsCommittee.activeUpToSlot(subnet); - } - /** Call ONLY ONCE: Two epoch before the fork, re-subscribe all existing random subscriptions to the new fork */ subscribeSubnetsToNextFork(nextFork: ForkName): void { this.logger.info("Suscribing to random attnets to next fork", {nextFork}); diff --git a/packages/beacon-node/src/network/subnets/dllAttnetsService.ts b/packages/beacon-node/src/network/subnets/dllAttnetsService.ts new file mode 100644 index 000000000000..ae3abb8bbf91 --- /dev/null +++ b/packages/beacon-node/src/network/subnets/dllAttnetsService.ts @@ -0,0 +1,315 @@ +import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; +import {ChainForkConfig} from "@lodestar/config"; +import { + ATTESTATION_SUBNET_COUNT, + EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION, + ForkName, + SLOTS_PER_EPOCH, +} from "@lodestar/params"; +import {Epoch, Slot, ssz} from "@lodestar/types"; +import {Logger, MapDef} from "@lodestar/utils"; +import {ClockEvent, IClock} from "../../util/clock.js"; +import {GossipType} from "../gossip/index.js"; +import {MetadataController} from "../metadata.js"; +import {SubnetMap, RequestedSubnet} from "../peers/utils/index.js"; +import {getActiveForks} from "../forks.js"; +import {NetworkCoreMetrics} from "../core/metrics.js"; +import {IAttnetsService, CommitteeSubscription, SubnetsServiceOpts, GossipSubscriber, NodeId} from "./interface.js"; +import {computeSubscribedSubnet} from "./util.js"; + +const gossipType = GossipType.beacon_attestation; + +enum SubnetSource { + committee = "committee", + longLived = "long_lived", +} + +/** + * Manage deleterministic long lived (DLL) subnets and short lived subnets. + * - PeerManager uses attnetsService to know which peers are required for duties and long lived subscriptions + * - Network call addCommitteeSubscriptions() from API calls + * - Gossip handler checks shouldProcess to know if validator is aggregator + */ +export class DLLAttnetsService implements IAttnetsService { + /** Committee subnets - PeerManager must find peers for those */ + private committeeSubnets = new SubnetMap(); + /** + * All currently subscribed short-lived subnets, for attestation aggregation + * This class will tell gossip to subscribe and un-subscribe + * If a value exists for `SubscriptionId` it means that gossip subscription is active in network.gossip + */ + private shortLivedSubscriptions = new SubnetMap(); + /** ${SUBNETS_PER_NODE} long lived subscriptions, may overlap with `shortLivedSubscriptions` */ + private longLivedSubscriptions = new Set(); + /** + * Map of an aggregator at a slot and subnet + * Used to determine if we should process an attestation. + */ + private aggregatorSlotSubnet = new MapDef>(() => new Set()); + + constructor( + private readonly config: ChainForkConfig, + private readonly clock: IClock, + private readonly gossip: GossipSubscriber, + private readonly metadata: MetadataController, + private readonly logger: Logger, + private readonly metrics: NetworkCoreMetrics | null, + private readonly nodeId: NodeId | null, + private readonly opts?: SubnetsServiceOpts + ) { + // if subscribeAllSubnets, we act like we have >= ATTESTATION_SUBNET_COUNT validators connecting to this node + // so that we have enough subnet topic peers, see https://github.com/ChainSafe/lodestar/issues/4921 + if (this.opts?.subscribeAllSubnets) { + for (let subnet = 0; subnet < ATTESTATION_SUBNET_COUNT; subnet++) { + this.committeeSubnets.request({subnet, toSlot: Infinity}); + } + } + + if (metrics) { + metrics.attnetsService.longLivedSubscriptions.addCollect(() => this.onScrapeLodestarMetrics(metrics)); + } + this.recomputeLongLivedSubnets(); + this.clock.on(ClockEvent.slot, this.onSlot); + this.clock.on(ClockEvent.epoch, this.onEpoch); + } + + close(): void { + this.clock.off(ClockEvent.slot, this.onSlot); + this.clock.off(ClockEvent.epoch, this.onEpoch); + } + + /** + * Get all active subnets for the hearbeat: + * - committeeSubnets so that submitted attestations can be spread to the network + * - longLivedSubscriptions because other peers based on this node's ENR for their submitted attestations + */ + getActiveSubnets(): RequestedSubnet[] { + const shortLivedSubnets = this.committeeSubnets.getActiveTtl(this.clock.currentSlot); + + const longLivedSubscriptionsToSlot = + (Math.floor(this.clock.currentEpoch / EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION) + 1) * + EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION * + SLOTS_PER_EPOCH; + const longLivedSubnets = Array.from(this.longLivedSubscriptions).map((subnet) => ({ + subnet, + toSlot: longLivedSubscriptionsToSlot, + })); + + // could be overlap, PeerDiscovery will handle it + return [...shortLivedSubnets, ...longLivedSubnets]; + } + + /** + * Called from the API when validator is a part of a committee. + */ + addCommitteeSubscriptions(subscriptions: CommitteeSubscription[]): void { + const subnetsToSubscribe: RequestedSubnet[] = []; + + for (const {subnet, slot, isAggregator} of subscriptions) { + // the peer-manager heartbeat will help find the subnet + this.committeeSubnets.request({subnet, toSlot: slot + 1}); + if (isAggregator) { + // need exact slot here + subnetsToSubscribe.push({subnet, toSlot: slot}); + this.aggregatorSlotSubnet.getOrDefault(slot).add(subnet); + } + } + + // Trigger gossip subscription first, in batch + if (subnetsToSubscribe.length > 0) { + this.subscribeToSubnets( + subnetsToSubscribe.map((sub) => sub.subnet), + SubnetSource.committee + ); + } + // Then, register the subscriptions + for (const subscription of subnetsToSubscribe) { + this.shortLivedSubscriptions.request(subscription); + } + } + + /** + * Check if a subscription is still active before handling a gossip object + */ + shouldProcess(subnet: number, slot: Slot): boolean { + if (!this.aggregatorSlotSubnet.has(slot)) { + return false; + } + return this.aggregatorSlotSubnet.getOrDefault(slot).has(subnet); + } + + /** + * TODO-dll: clarify how many epochs before the fork we should subscribe to the new fork + * Call ONLY ONCE: Two epoch before the fork, re-subscribe all existing random subscriptions to the new fork + **/ + subscribeSubnetsToNextFork(nextFork: ForkName): void { + this.logger.info("Suscribing to long lived attnets to next fork", { + nextFork, + subnets: Array.from(this.longLivedSubscriptions).join(","), + }); + for (const subnet of this.longLivedSubscriptions) { + this.gossip.subscribeTopic({type: gossipType, fork: nextFork, subnet}); + } + } + + /** + * TODO-dll: clarify how many epochs after the fork we should unsubscribe to the new fork + * Call ONLY ONCE: Two epochs after the fork, un-subscribe all subnets from the old fork + **/ + unsubscribeSubnetsFromPrevFork(prevFork: ForkName): void { + this.logger.info("Unsuscribing to long lived attnets from prev fork", {prevFork}); + for (let subnet = 0; subnet < ATTESTATION_SUBNET_COUNT; subnet++) { + if (!this.opts?.subscribeAllSubnets) { + this.gossip.unsubscribeTopic({type: gossipType, fork: prevFork, subnet}); + } + } + } + + /** + * Run per slot. + */ + private onSlot = (slot: Slot): void => { + try { + this.unsubscribeExpiredCommitteeSubnets(slot); + } catch (e) { + this.logger.error("Error on AttnetsService.onSlot", {slot}, e as Error); + } + }; + + /** + * Run per epoch, clean-up operations that are not urgent + * Subscribe to new random subnets every EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION epochs + */ + private onEpoch = (epoch: Epoch): void => { + try { + if (epoch % EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION === 0) { + this.recomputeLongLivedSubnets(); + } + const slot = computeStartSlotAtEpoch(epoch); + this.pruneExpiredAggregator(slot); + } catch (e) { + this.logger.error("Error on AttnetsService.onEpoch", {epoch}, e as Error); + } + }; + + private recomputeLongLivedSubnets(): void { + if (this.nodeId === null) { + this.logger.verbose("Cannot recompute long-lived subscriptions, no nodeId"); + return; + } + + const oldSubnets = this.longLivedSubscriptions; + const newSubnets = computeSubscribedSubnet(this.nodeId, this.clock.currentEpoch); + this.logger.verbose("Recomputing long-lived subscriptions", { + epoch: this.clock.currentEpoch, + oldSubnets: Array.from(oldSubnets).join(","), + newSubnets: newSubnets.join(","), + }); + + const toRemoveSubnets = []; + for (const subnet of oldSubnets) { + if (!newSubnets.includes(subnet)) { + toRemoveSubnets.push(subnet); + } + } + + // First, tell gossip to subscribe to the subnets if not connected already + this.subscribeToSubnets(newSubnets, SubnetSource.longLived); + + // then update longLivedSubscriptions + for (const subnet of toRemoveSubnets) { + this.longLivedSubscriptions.delete(subnet); + } + + for (const subnet of newSubnets) { + // this.longLivedSubscriptions is a set so it'll handle duplicates + this.longLivedSubscriptions.add(subnet); + } + + // Only tell gossip to unsubsribe last, longLivedSubscriptions has the latest state + this.unsubscribeSubnets(toRemoveSubnets, this.clock.currentSlot, SubnetSource.longLived); + this.updateMetadata(); + } + + /** + * Unsubscribe to a committee subnet from subscribedCommitteeSubnets. + * If a random subnet is present, we do not unsubscribe from it. + */ + private unsubscribeExpiredCommitteeSubnets(slot: Slot): void { + const expired = this.shortLivedSubscriptions.getExpired(slot); + if (expired.length > 0) { + this.unsubscribeSubnets(expired, slot, SubnetSource.committee); + } + } + + /** + * No need to track aggregator for past slots. + * @param currentSlot + */ + private pruneExpiredAggregator(currentSlot: Slot): void { + for (const slot of this.aggregatorSlotSubnet.keys()) { + if (currentSlot > slot) { + this.aggregatorSlotSubnet.delete(slot); + } + } + } + + /** Update ENR */ + private updateMetadata(): void { + const subnets = ssz.phase0.AttestationSubnets.defaultValue(); + for (const subnet of this.longLivedSubscriptions) { + subnets.set(subnet, true); + } + + // Only update metadata if necessary, setting `metadata.[key]` triggers a write to disk + if (!ssz.phase0.AttestationSubnets.equals(subnets, this.metadata.attnets)) { + this.metadata.attnets = subnets; + } + } + + /** + * Trigger a gossip subcription only if not already subscribed + * shortLivedSubscriptions or longLivedSubscriptions should be updated right AFTER this called + **/ + private subscribeToSubnets(subnets: number[], src: SubnetSource): void { + const forks = getActiveForks(this.config, this.clock.currentEpoch); + for (const subnet of subnets) { + if (!this.shortLivedSubscriptions.has(subnet) && !this.longLivedSubscriptions.has(subnet)) { + for (const fork of forks) { + this.gossip.subscribeTopic({type: gossipType, fork, subnet}); + } + this.metrics?.attnetsService.subscribeSubnets.inc({subnet, src}); + } + } + } + + /** + * Trigger a gossip un-subscription only if no-one is still subscribed + * If unsubscribe long lived subnets, longLivedSubscriptions should be updated right BEFORE this called + **/ + private unsubscribeSubnets(subnets: number[], slot: Slot, src: SubnetSource): void { + // No need to unsubscribeTopic(). Return early to prevent repetitive extra work + if (this.opts?.subscribeAllSubnets) return; + + const forks = getActiveForks(this.config, this.clock.currentEpoch); + for (const subnet of subnets) { + if (!this.shortLivedSubscriptions.isActiveAtSlot(subnet, slot) && !this.longLivedSubscriptions.has(subnet)) { + for (const fork of forks) { + this.gossip.unsubscribeTopic({type: gossipType, fork, subnet}); + } + this.metrics?.attnetsService.unsubscribeSubnets.inc({subnet, src}); + } + } + } + + private onScrapeLodestarMetrics(metrics: NetworkCoreMetrics): void { + metrics.attnetsService.committeeSubnets.set(this.committeeSubnets.size); + metrics.attnetsService.subscriptionsCommittee.set(this.shortLivedSubscriptions.size); + metrics.attnetsService.longLivedSubscriptions.set(this.longLivedSubscriptions.size); + let aggregatorCount = 0; + for (const subnets of this.aggregatorSlotSubnet.values()) { + aggregatorCount += subnets.size; + } + metrics.attnetsService.aggregatorSlotSubnetCount.set(aggregatorCount); + } +} diff --git a/packages/beacon-node/src/network/subnets/interface.ts b/packages/beacon-node/src/network/subnets/interface.ts index 8dc56fc54c54..9f449427377f 100644 --- a/packages/beacon-node/src/network/subnets/interface.ts +++ b/packages/beacon-node/src/network/subnets/interface.ts @@ -1,5 +1,5 @@ import {ForkName} from "@lodestar/params"; -import {Slot, ValidatorIndex} from "@lodestar/types"; +import {Bytes32, Slot, ValidatorIndex} from "@lodestar/types"; import {RequestedSubnet} from "../peers/utils/index.js"; import {GossipTopic} from "../gossip/interface.js"; @@ -27,7 +27,11 @@ export type RandBetweenFn = (min: number, max: number) => number; export type ShuffleFn = (arr: T[]) => T[]; export type SubnetsServiceOpts = { + deterministicLongLivedAttnets?: boolean; subscribeAllSubnets?: boolean; +}; + +export type SubnetsServiceTestOpts = { // For deterministic randomness in unit test after ESM prevents simple import mocking randBetweenFn?: RandBetweenFn; shuffleFn?: ShuffleFn; @@ -37,3 +41,6 @@ export type GossipSubscriber = { subscribeTopic(topic: GossipTopic): void; unsubscribeTopic(topic: GossipTopic): void; }; + +// uint256 in the spec +export type NodeId = Bytes32; diff --git a/packages/beacon-node/src/network/subnets/util.ts b/packages/beacon-node/src/network/subnets/util.ts new file mode 100644 index 000000000000..e983209ba6d6 --- /dev/null +++ b/packages/beacon-node/src/network/subnets/util.ts @@ -0,0 +1,60 @@ +import {digest} from "@chainsafe/as-sha256"; +import { + ATTESTATION_SUBNET_PREFIX_BITS, + EPOCHS_PER_SUBNET_SUBSCRIPTION, + NODE_ID_BITS, + SUBNETS_PER_NODE, +} from "@lodestar/params"; +import {ATTESTATION_SUBNET_COUNT} from "@lodestar/params"; +import {computeShuffledIndex} from "@lodestar/state-transition"; +import {Epoch, ssz} from "@lodestar/types"; +import {NodeId} from "./interface.js"; + +/** + * Spec https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/phase0/p2p-interface.md + */ +export function computeSubscribedSubnet(nodeId: NodeId, epoch: Epoch): number[] { + const subnets: number[] = []; + for (let index = 0; index < SUBNETS_PER_NODE; index++) { + subnets.push(computeSubscribedSubnetByIndex(nodeId, epoch, index)); + } + return subnets; +} + +/** + * Spec https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/phase0/p2p-interface.md + */ +export function computeSubscribedSubnetByIndex(nodeId: NodeId, epoch: Epoch, index: number): number { + const nodeIdPrefix = getNodeIdPrefix(nodeId); + const nodeOffset = getNodeOffset(nodeId); + const permutationSeed = digest( + ssz.UintNum64.serialize(Math.floor((epoch + nodeOffset) / EPOCHS_PER_SUBNET_SUBSCRIPTION)) + ); + const permutatedPrefix = computeShuffledIndex(nodeIdPrefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutationSeed); + return (permutatedPrefix + index) % ATTESTATION_SUBNET_COUNT; +} + +/** + * Should return node_id >> (NODE_ID_BITS - int(ATTESTATION_SUBNET_PREFIX_BITS)) + * Ideally we should use bigint here but since these constants are not likely to change we can use number + */ +export function getNodeIdPrefix(nodeId: NodeId): number { + const totalShiftedBits = NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS; + const shiftedBytes = Math.floor(totalShiftedBits / 8); + const shiftedBits = totalShiftedBits % 8; + const prefixBytes = nodeId.slice(0, nodeId.length - shiftedBytes); + const dataView = new DataView(prefixBytes.buffer, prefixBytes.byteOffset, prefixBytes.byteLength); + // only 6 bits are used for prefix so getUint8() is safe + const prefix = dataView.getUint8(0) >> shiftedBits; + return prefix; +} + +/** + * Should return node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION + * This function is safe to return number because EPOCHS_PER_SUBNET_SUBSCRIPTION is 256 + */ +export function getNodeOffset(nodeId: NodeId): number { + // Big endian means that the least significant byte comes last + // The n % 256 is equivalent to the last byte of the node_id + return nodeId[nodeId.length - 1]; +} diff --git a/packages/beacon-node/test/unit-mainnet/network/subnets/util.test.ts b/packages/beacon-node/test/unit-mainnet/network/subnets/util.test.ts new file mode 100644 index 000000000000..b99d2417dd9d --- /dev/null +++ b/packages/beacon-node/test/unit-mainnet/network/subnets/util.test.ts @@ -0,0 +1,67 @@ +import {expect} from "chai"; +import {bigIntToBytes} from "@lodestar/utils"; +import {computeSubscribedSubnet} from "../../../../src/network/subnets/util.js"; + +describe("computeSubscribedSubnet", () => { + // lighthouse's test cases https://github.com/sigp/lighthouse/blob/cc780aae3e0cb89649086a3b63cb02a4f97f7ae2/consensus/types/src/subnet_id.rs#L169 + // this goes with mainnet config + const testCases: {nodeId: string; epoch: number; expected: number[]}[] = [ + { + nodeId: "0", + epoch: 54321, + expected: [4, 5], + }, + { + nodeId: "88752428858350697756262172400162263450541348766581994718383409852729519486397", + epoch: 1017090249, + expected: [61, 62], + }, + { + nodeId: "18732750322395381632951253735273868184515463718109267674920115648614659369468", + epoch: 1827566880, + expected: [23, 24], + }, + { + nodeId: "27726842142488109545414954493849224833670205008410190955613662332153332462900", + epoch: 846255942, + expected: [38, 39], + }, + { + nodeId: "39755236029158558527862903296867805548949739810920318269566095185775868999998", + epoch: 766597383, + expected: [53, 54], + }, + { + nodeId: "31899136003441886988955119620035330314647133604576220223892254902004850516297", + epoch: 1204990115, + expected: [39, 40], + }, + { + nodeId: "58579998103852084482416614330746509727562027284701078483890722833654510444626", + epoch: 1616209495, + expected: [48, 49], + }, + { + nodeId: "28248042035542126088870192155378394518950310811868093527036637864276176517397", + epoch: 1774367616, + expected: [39, 40], + }, + { + nodeId: "60930578857433095740782970114409273483106482059893286066493409689627770333527", + epoch: 1484598751, + expected: [34, 35], + }, + { + nodeId: "103822458477361691467064888613019442068586830412598673713899771287914656699997", + epoch: 3525502229, + expected: [37, 38], + }, + ]; + + for (const [index, {nodeId, epoch, expected}] of testCases.entries()) { + it(`test case ${index}`, () => { + // node is is of type uint256 = 32 bytes + expect(computeSubscribedSubnet(bigIntToBytes(BigInt(nodeId), 32, "be"), epoch)).to.deep.equal(expected); + }); + } +}); diff --git a/packages/beacon-node/test/unit/network/subnets/dllAttnetsService.test.ts b/packages/beacon-node/test/unit/network/subnets/dllAttnetsService.test.ts new file mode 100644 index 000000000000..891c87918a91 --- /dev/null +++ b/packages/beacon-node/test/unit/network/subnets/dllAttnetsService.test.ts @@ -0,0 +1,149 @@ +import {expect} from "chai"; +import sinon, {SinonStubbedInstance} from "sinon"; +import {createBeaconConfig} from "@lodestar/config"; +import {ZERO_HASH} from "@lodestar/state-transition"; +import { + ATTESTATION_SUBNET_COUNT, + EPOCHS_PER_SUBNET_SUBSCRIPTION, + ForkName, + SLOTS_PER_EPOCH, + SUBNETS_PER_NODE, +} from "@lodestar/params"; +import {getCurrentSlot} from "@lodestar/state-transition"; +import {bigIntToBytes} from "@lodestar/utils"; +import {Clock, IClock} from "../../../../src/util/clock.js"; +import {Eth2Gossipsub} from "../../../../src/network/gossip/gossipsub.js"; +import {MetadataController} from "../../../../src/network/metadata.js"; +import {testLogger} from "../../../utils/logger.js"; +import {DLLAttnetsService} from "../../../../src/network/subnets/dllAttnetsService.js"; +import {CommitteeSubscription} from "../../../../src/network/subnets/interface.js"; + +describe("DLLAttnetsService", () => { + const nodeId = bigIntToBytes( + BigInt("88752428858350697756262172400162263450541348766581994718383409852729519486397"), + 32, + "be" + ); + const ALTAIR_FORK_EPOCH = 100; + // eslint-disable-next-line @typescript-eslint/naming-convention + const config = createBeaconConfig({ALTAIR_FORK_EPOCH}, ZERO_HASH); + // const {SECONDS_PER_SLOT} = config; + let service: DLLAttnetsService; + const sandbox = sinon.createSandbox(); + let gossipStub: SinonStubbedInstance & Eth2Gossipsub; + let metadata: MetadataController; + + let clock: IClock; + const logger = testLogger(); + + beforeEach(function () { + sandbox.useFakeTimers(Date.now()); + gossipStub = sandbox.createStubInstance(Eth2Gossipsub) as SinonStubbedInstance & Eth2Gossipsub; + clock = new Clock({ + genesisTime: Math.floor(Date.now() / 1000), + config, + signal: new AbortController().signal, + }); + + // load getCurrentSlot first, vscode not able to debug without this + getCurrentSlot(config, Math.floor(Date.now() / 1000)); + metadata = new MetadataController({}, {config, onSetValue: () => null}); + service = new DLLAttnetsService(config, clock, gossipStub, metadata, logger, null, nodeId); + }); + + afterEach(() => { + service.close(); + sandbox.restore(); + }); + + it("should subscribe to deterministic long lived subnets on constructor", () => { + expect(gossipStub.subscribeTopic.calledTwice).to.be.true; + }); + + it("should change long lived subnets after EPOCHS_PER_SUBNET_SUBSCRIPTION", () => { + expect(gossipStub.subscribeTopic.calledTwice).to.be.true; + expect(gossipStub.subscribeTopic.callCount).to.be.equal(SUBNETS_PER_NODE); + sandbox.clock.tick(config.SECONDS_PER_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SUBNET_SUBSCRIPTION * 1000); + // SUBNETS_PER_NODE = 2 => 2 more calls + expect(gossipStub.subscribeTopic.callCount).to.be.equal(2 * SUBNETS_PER_NODE); + }); + + it("should subscribe to new fork 2 epochs before ALTAIR_FORK_EPOCH", () => { + expect(gossipStub.subscribeTopic.calledWithMatch({fork: ForkName.phase0})).to.be.true; + expect(gossipStub.subscribeTopic.calledWithMatch({fork: ForkName.altair})).to.be.false; + expect(gossipStub.subscribeTopic.calledTwice).to.be.true; + const firstSubnet = (gossipStub.subscribeTopic.args[0][0] as unknown as {subnet: number}).subnet; + const secondSubnet = (gossipStub.subscribeTopic.args[1][0] as unknown as {subnet: number}).subnet; + expect(gossipStub.subscribeTopic.callCount).to.be.equal(SUBNETS_PER_NODE); + sandbox.clock.tick(config.SECONDS_PER_SLOT * SLOTS_PER_EPOCH * (ALTAIR_FORK_EPOCH - 2) * 1000); + service.subscribeSubnetsToNextFork(ForkName.altair); + // SUBNETS_PER_NODE = 2 => 2 more calls + // same subnets were called + expect(gossipStub.subscribeTopic.calledWithMatch({fork: ForkName.altair, subnet: firstSubnet})).to.be.true; + expect(gossipStub.subscribeTopic.calledWithMatch({fork: ForkName.altair, subnet: secondSubnet})).to.be.true; + expect(gossipStub.subscribeTopic.callCount).to.be.equal(2 * SUBNETS_PER_NODE); + // 2 epochs after the fork + sandbox.clock.tick(config.SECONDS_PER_SLOT * 4 * 1000); + service.unsubscribeSubnetsFromPrevFork(ForkName.phase0); + expect(gossipStub.unsubscribeTopic.calledWithMatch({fork: ForkName.phase0, subnet: firstSubnet})).to.be.true; + expect(gossipStub.unsubscribeTopic.calledWithMatch({fork: ForkName.phase0, subnet: secondSubnet})).to.be.true; + expect(gossipStub.unsubscribeTopic.callCount).to.be.equal(ATTESTATION_SUBNET_COUNT); + }); + + it("should not subscribe to new short lived subnet if not aggregator", () => { + expect(gossipStub.subscribeTopic.callCount).to.be.equal(SUBNETS_PER_NODE); + const firstSubnet = (gossipStub.subscribeTopic.args[0][0] as unknown as {subnet: number}).subnet; + const secondSubnet = (gossipStub.subscribeTopic.args[1][0] as unknown as {subnet: number}).subnet; + // should subscribe to new short lived subnet + const newSubnet = 63; + expect(newSubnet).to.be.not.equal(firstSubnet); + expect(newSubnet).to.be.not.equal(secondSubnet); + const subscription: CommitteeSubscription = { + validatorIndex: 2023, + subnet: newSubnet, + slot: 100, + isAggregator: false, + }; + service.addCommitteeSubscriptions([subscription]); + // no new subscription + expect(gossipStub.subscribeTopic.callCount).to.be.equal(SUBNETS_PER_NODE); + }); + + it("should subscribe to new short lived subnet if aggregator", () => { + expect(gossipStub.subscribeTopic.callCount).to.be.equal(SUBNETS_PER_NODE); + const firstSubnet = (gossipStub.subscribeTopic.args[0][0] as unknown as {subnet: number}).subnet; + const secondSubnet = (gossipStub.subscribeTopic.args[1][0] as unknown as {subnet: number}).subnet; + // should subscribe to new short lived subnet + const newSubnet = 63; + expect(newSubnet).to.be.not.equal(firstSubnet); + expect(newSubnet).to.be.not.equal(secondSubnet); + const subscription: CommitteeSubscription = { + validatorIndex: 2023, + subnet: newSubnet, + slot: 100, + isAggregator: true, + }; + service.addCommitteeSubscriptions([subscription]); + expect(gossipStub.subscribeTopic.callCount).to.be.equal(SUBNETS_PER_NODE + 1); + // then unsubscribe after the expiration + sandbox.clock.tick(config.SECONDS_PER_SLOT * (subscription.slot + 1) * 1000); + expect(gossipStub.unsubscribeTopic.calledWithMatch({subnet: newSubnet})).to.be.true; + }); + + it("should not subscribe to existing short lived subnet if aggregator", () => { + expect(gossipStub.subscribeTopic.callCount).to.be.equal(SUBNETS_PER_NODE); + const firstSubnet = (gossipStub.subscribeTopic.args[0][0] as unknown as {subnet: number}).subnet; + // should not subscribe to existing short lived subnet + const subscription: CommitteeSubscription = { + validatorIndex: 2023, + subnet: firstSubnet, + slot: 100, + isAggregator: true, + }; + service.addCommitteeSubscriptions([subscription]); + expect(gossipStub.subscribeTopic.callCount).to.be.equal(SUBNETS_PER_NODE); + // then should not subscribe after the expiration + sandbox.clock.tick(config.SECONDS_PER_SLOT * (subscription.slot + 1) * 1000); + expect(gossipStub.unsubscribeTopic.called).to.be.false; + }); +}); diff --git a/packages/beacon-node/test/unit/network/subnets/util.test.ts b/packages/beacon-node/test/unit/network/subnets/util.test.ts new file mode 100644 index 000000000000..fbaad75f4810 --- /dev/null +++ b/packages/beacon-node/test/unit/network/subnets/util.test.ts @@ -0,0 +1,41 @@ +import {expect} from "chai"; +import {bigIntToBytes} from "@lodestar/utils"; +import {ATTESTATION_SUBNET_PREFIX_BITS, NODE_ID_BITS} from "@lodestar/params"; +import {getNodeIdPrefix, getNodeOffset} from "../../../../src/network/subnets/util.js"; + +const nodeIds: string[] = [ + "0", + "88752428858350697756262172400162263450541348766581994718383409852729519486397", + "18732750322395381632951253735273868184515463718109267674920115648614659369468", + "27726842142488109545414954493849224833670205008410190955613662332153332462900", + "39755236029158558527862903296867805548949739810920318269566095185775868999998", + "31899136003441886988955119620035330314647133604576220223892254902004850516297", + "58579998103852084482416614330746509727562027284701078483890722833654510444626", + "28248042035542126088870192155378394518950310811868093527036637864276176517397", + "60930578857433095740782970114409273483106482059893286066493409689627770333527", + "103822458477361691467064888613019442068586830412598673713899771287914656699997", +]; + +describe("getNodeIdPrefix", () => { + for (const [index, nodeId] of nodeIds.entries()) { + it(`test case ${index}`, () => { + const nodeIdBigInt = BigInt(nodeId); + // nodeId is of type uint256, which is 32 bytes + const nodeIdBytes = bigIntToBytes(nodeIdBigInt, 32, "be"); + expect(getNodeIdPrefix(nodeIdBytes)).to.equal( + Number(nodeIdBigInt >> BigInt(NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS)) + ); + }); + } +}); + +describe("getNodeOffset", () => { + for (const [index, nodeId] of nodeIds.entries()) { + it(`test case ${index}`, () => { + const nodeIdBigInt = BigInt(nodeId); + // nodeId is of type uint256, which is 32 bytes + const nodeIdBytes = bigIntToBytes(nodeIdBigInt, 32, "be"); + expect(getNodeOffset(nodeIdBytes)).to.equal(Number(nodeIdBigInt % BigInt(256))); + }); + } +}); diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index b822e270fe7c..af6f27884187 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -11,6 +11,7 @@ export type NetworkArgs = { discoveryPort?: number; bootnodes?: string[]; targetPeers?: number; + deterministicLongLivedAttnets?: boolean; subscribeAllSubnets?: boolean; disablePeerScoring?: boolean; mdns?: boolean; @@ -63,6 +64,7 @@ export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { maxPeers: maxPeers ?? defaultOptions.network.maxPeers, targetPeers: targetPeers ?? defaultOptions.network.targetPeers, localMultiaddrs: [`/ip4/${listenAddress}/tcp/${tcpPort}`], + deterministicLongLivedAttnets: args["deterministicLongLivedAttnets"], subscribeAllSubnets: args["subscribeAllSubnets"], disablePeerScoring: args["disablePeerScoring"], connectToDiscv5Bootnodes: args["network.connectToDiscv5Bootnodes"], @@ -129,6 +131,13 @@ export const options: CliCommandOptions = { group: "network", }, + deterministicLongLivedAttnets: { + type: "boolean", + description: "Use deterministic subnet selection for long-lived subnet subscriptions", + defaultDescription: String(defaultOptions.network.deterministicLongLivedAttnets === true), + group: "network", + }, + subscribeAllSubnets: { type: "boolean", description: "Subscribe to all subnets regardless of validator count", diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index 9307424a1b46..acf279a9a160 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -71,6 +71,7 @@ describe("options / beaconNodeOptions", () => { discoveryPort: 9002, bootnodes: ["enr:-somedata"], targetPeers: 25, + deterministicLongLivedAttnets: true, subscribeAllSubnets: true, disablePeerScoring: true, mdns: false, @@ -171,6 +172,7 @@ describe("options / beaconNodeOptions", () => { maxPeers: 30, targetPeers: 25, localMultiaddrs: ["/ip4/127.0.0.1/tcp/9001"], + deterministicLongLivedAttnets: true, subscribeAllSubnets: true, disablePeerScoring: true, connectToDiscv5Bootnodes: true, diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index d8e8137796d5..b167f25ffd97 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -174,6 +174,10 @@ export const RANDOM_SUBNETS_PER_VALIDATOR = 1; export const EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION = 256; /** Rationale: https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#why-are-there-attestation_subnet_count-attestation-subnets */ export const ATTESTATION_SUBNET_COUNT = 64; +export const SUBNETS_PER_NODE = 2; +export const NODE_ID_BITS = 256; +export const ATTESTATION_SUBNET_PREFIX_BITS = Math.log2(ATTESTATION_SUBNET_COUNT); +export const EPOCHS_PER_SUBNET_SUBSCRIPTION = 256; // altair validator From 7e34c46de5e4046eff5bdab045eba0a5bd65ff7a Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Jul 2023 15:51:45 +0200 Subject: [PATCH 54/96] chore: re-add explicit exit after beacon node closed (#5748) Re-add explicit exit after beacon node closed This reverts commit 1de881b7776284c6dd56f146ac55ef3df040f98d. --- packages/cli/src/cmds/beacon/handler.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cli/src/cmds/beacon/handler.ts b/packages/cli/src/cmds/beacon/handler.ts index af34cf861697..4c71e4714dd2 100644 --- a/packages/cli/src/cmds/beacon/handler.ts +++ b/packages/cli/src/cmds/beacon/handler.ts @@ -130,6 +130,9 @@ export async function beaconHandler(args: BeaconArgs & GlobalArgs): Promise Date: Wed, 12 Jul 2023 18:35:07 +0200 Subject: [PATCH 55/96] feat: track attestationInBlockParticipants (#5749) Track attestationInBlockParticipants --- packages/beacon-node/src/metrics/metrics/lodestar.ts | 5 +++++ packages/beacon-node/src/metrics/validatorMonitor.ts | 3 +++ 2 files changed, 8 insertions(+) diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index fb2d540abb45..082f3680e161 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -850,6 +850,11 @@ export function createLodestarMetrics( help: "The excess slots (beyond the minimum delay) between the attestation slot and the block slot", buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10], }), + attestationInBlockParticipants: register.histogram({ + name: "validator_monitor_attestation_in_block_participants", + help: "The total participants in attestations of monitored validators included in blocks", + buckets: [1, 5, 20, 50, 100, 200], + }), syncSignatureInAggregateTotal: register.gauge({ name: "validator_monitor_sync_signature_in_aggregate_total", help: "Number of times a sync signature has been seen in an aggregate", diff --git a/packages/beacon-node/src/metrics/validatorMonitor.ts b/packages/beacon-node/src/metrics/validatorMonitor.ts index 5b0a5e1d5089..83d0647ea7fc 100644 --- a/packages/beacon-node/src/metrics/validatorMonitor.ts +++ b/packages/beacon-node/src/metrics/validatorMonitor.ts @@ -529,12 +529,14 @@ export function createValidatorMonitor( const inclusionDistance = Math.max(parentSlot - data.slot, 0) + 1; const delay = inclusionDistance - MIN_ATTESTATION_INCLUSION_DELAY; const epoch = computeEpochAtSlot(data.slot); + const participants = indexedAttestation.attestingIndices.length; for (const index of indexedAttestation.attestingIndices) { const validator = validators.get(index); if (validator) { metrics.validatorMonitor.attestationInBlockTotal.inc(); metrics.validatorMonitor.attestationInBlockDelaySlots.observe(delay); + metrics.validatorMonitor.attestationInBlockParticipants.observe(participants); const summary = getEpochSummary(validator, epoch); summary.attestationBlockInclusions += 1; @@ -566,6 +568,7 @@ export function createValidatorMonitor( committeeIndex: data.index, inclusionDistance, correctHead, + participants, }); } } From 5fa2c8821974698be898e4db70c213d43bfc5985 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Thu, 13 Jul 2023 01:06:01 +0200 Subject: [PATCH 56/96] fix: fix eslint for import/no-extraneous-dependencies (#5752) * Update eslint dependencies * Fix auto-fixed linting issues * Fix the import linting rule * Fix import related linting issues * Remove unused code * Add reference to an issue * Fix import order linting * Fix relative import paths * Fix relative import issue * Fix relative import issue * Fix relative import issue --- .eslintrc.js | 12 +- package.json | 11 +- packages/api/src/beacon/client/proof.ts | 2 +- packages/api/src/beacon/routes/config.ts | 2 +- packages/api/src/beacon/routes/debug.ts | 2 +- packages/api/src/beacon/routes/events.ts | 2 +- packages/api/src/beacon/routes/node.ts | 2 +- packages/api/src/beacon/server/proof.ts | 2 +- packages/api/src/builder/routes.ts | 2 +- packages/api/src/interfaces.ts | 2 +- packages/api/src/utils/client/client.ts | 4 +- packages/api/src/utils/schema.ts | 2 +- .../api/src/utils/server/genericJsonServer.ts | 4 +- packages/api/src/utils/types.ts | 4 +- .../beacon/genericServerTest/debug.test.ts | 2 +- .../api/test/unit/beacon/testData/beacon.ts | 2 +- .../api/test/unit/beacon/testData/debug.ts | 2 +- .../test/unit/beacon/testData/lightclient.ts | 2 +- packages/api/test/unit/builder/testData.ts | 2 +- packages/api/test/utils/genericServerTest.ts | 2 +- packages/beacon-node/package.json | 4 +- .../src/api/impl/beacon/blocks/index.ts | 2 +- .../src/api/impl/beacon/state/utils.ts | 2 +- .../src/api/impl/lightclient/index.ts | 2 +- .../src/api/impl/lodestar/index.ts | 2 +- .../beacon-node/src/api/impl/proof/index.ts | 2 +- .../src/api/impl/validator/index.ts | 2 +- packages/beacon-node/src/api/rest/base.ts | 5 +- .../src/chain/beaconProposerCache.ts | 5 +- .../src/chain/blocks/importBlock.ts | 2 +- .../src/chain/blocks/verifyBlock.ts | 2 +- .../blocks/verifyBlocksExecutionPayloads.ts | 4 +- packages/beacon-node/src/chain/chain.ts | 2 +- .../src/chain/errors/attestationError.ts | 2 +- .../src/chain/errors/blockError.ts | 7 +- .../beacon-node/src/chain/genesis/genesis.ts | 2 +- .../src/chain/genesis/interface.ts | 2 +- packages/beacon-node/src/chain/initState.ts | 2 +- packages/beacon-node/src/chain/interface.ts | 2 +- .../src/chain/lightClient/index.ts | 7 +- .../src/chain/lightClient/proofs.ts | 2 +- .../opPools/aggregatedAttestationPool.ts | 7 +- .../src/chain/opPools/attestationPool.ts | 2 +- .../beacon-node/src/chain/opPools/opPool.ts | 4 +- .../chain/opPools/syncCommitteeMessagePool.ts | 2 +- .../opPools/syncContributionAndProofPool.ts | 2 +- .../chain/produceBlock/computeNewStateRoot.ts | 3 - .../beacon-node/src/chain/regen/queued.ts | 2 +- packages/beacon-node/src/chain/regen/regen.ts | 2 +- .../chain/seenCache/seenAggregateAndProof.ts | 2 +- .../seenCache/seenCommitteeContribution.ts | 2 +- .../src/chain/validation/attestation.ts | 2 +- .../beacon-node/src/chain/validation/block.ts | 2 +- .../signatureSets/aggregateAndProof.ts | 2 +- .../signatureSets/selectionProof.ts | 2 +- .../src/chain/validation/syncCommittee.ts | 2 +- packages/beacon-node/src/db/beacon.ts | 5 +- packages/beacon-node/src/db/buckets.ts | 3 + .../src/db/repositories/blobSidecars.ts | 2 +- .../lightclientSyncCommitteeWitness.ts | 2 +- .../beacon-node/src/eth1/eth1DepositsCache.ts | 2 +- .../src/eth1/eth1MergeBlockTracker.ts | 2 +- packages/beacon-node/src/eth1/index.ts | 2 +- .../src/eth1/provider/jsonRpcHttpClient.ts | 5 +- .../beacon-node/src/eth1/provider/utils.ts | 2 +- .../beacon-node/src/eth1/utils/deposits.ts | 4 +- .../beacon-node/src/execution/builder/http.ts | 2 +- .../src/execution/builder/index.ts | 2 +- .../beacon-node/src/execution/engine/http.ts | 5 +- .../beacon-node/src/execution/engine/index.ts | 2 +- .../beacon-node/src/execution/engine/types.ts | 4 +- .../src/network/core/networkCore.ts | 8 +- .../src/network/core/networkCoreWorker.ts | 2 +- .../network/core/networkCoreWorkerHandler.ts | 8 +- .../beacon-node/src/network/core/types.ts | 4 +- .../beacon-node/src/network/discv5/index.ts | 5 +- .../beacon-node/src/network/discv5/utils.ts | 2 +- .../src/network/gossip/encoding.ts | 7 +- packages/beacon-node/src/network/network.ts | 2 +- .../beacon-node/src/network/peers/discover.ts | 4 +- .../peers/utils/assertPeerRelevance.ts | 2 +- .../network/peers/utils/prioritizePeers.ts | 2 +- .../src/network/processor/index.ts | 5 +- .../src/network/reqresp/utils/collect.ts | 2 +- packages/beacon-node/src/node/nodejs.ts | 2 +- .../src/node/utils/interop/deposits.ts | 2 +- .../beacon-node/src/sync/backfill/backfill.ts | 4 +- packages/beacon-node/src/sync/range/chain.ts | 2 +- .../src/sync/range/utils/chainTarget.ts | 2 +- packages/beacon-node/src/sync/unknownBlock.ts | 2 +- .../e2e/api/impl/lightclient/endpoint.test.ts | 2 +- .../test/e2e/chain/lightclient.test.ts | 4 +- .../e2e/doppelganger/doppelganger.test.ts | 2 +- .../e2e/eth1/eth1ForBlockProduction.test.ts | 2 +- .../e2e/eth1/eth1MergeBlockTracker.test.ts | 2 +- .../e2e/network/peers/peerManager.test.ts | 2 +- .../test/e2e/sync/unknownBlockSync.test.ts | 2 +- .../perf/api/impl/validator/attester.test.ts | 1 + .../opPools/aggregatedAttestationPool.test.ts | 5 +- .../seenCache/seenAggregateAndProof.test.ts | 2 +- .../validation/aggregateAndProof.test.ts | 3 +- .../perf/chain/validation/attestation.test.ts | 3 +- .../perf/chain/verifyImportBlocks.test.ts | 4 +- .../test/perf/eth1/pickEth1Vote.test.ts | 2 +- .../beacon-node/test/perf/misc/throw.test.ts | 5 +- .../test/scripts/blsPubkeyBytesFrequency.ts | 2 +- packages/beacon-node/test/spec/bls/bls.ts | 2 +- packages/beacon-node/test/spec/general/bls.ts | 2 +- .../test/spec/presets/fork_choice.ts | 2 +- .../light_client/single_merkle_proof.ts | 4 +- .../test/spec/presets/light_client/sync.ts | 2 +- .../beacon-node/test/spec/presets/merkle.ts | 4 +- .../beacon-node/test/spec/presets/rewards.ts | 2 +- .../test/spec/presets/ssz_static.ts | 2 +- .../beacon/blocks/getBlockHeaders.test.ts | 2 +- .../unit/api/impl/beacon/state/utils.test.ts | 2 +- .../unit/chain/archive/blockArchiver.test.ts | 2 +- .../test/unit/chain/genesis/genesis.test.ts | 2 +- .../unit/chain/opPools/syncCommittee.test.ts | 2 +- .../opPools/syncCommitteeContribution.test.ts | 4 +- .../stateCache/stateContextCache.test.ts | 2 +- .../validation/aggregateAndProof.test.ts | 3 +- .../unit/chain/validation/attestation.test.ts | 5 +- .../validation/blsToExecutionChange.test.ts | 4 +- .../chain/validation/syncCommittee.test.ts | 2 +- .../chain/validation/voluntaryExit.test.ts | 4 +- .../unit/eth1/eth1MergeBlockTracker.test.ts | 2 +- .../test/unit/eth1/utils/deposits.test.ts | 2 +- .../test/unit/eth1/utils/eth1Data.test.ts | 2 +- .../unit/executionEngine/httpRetry.test.ts | 2 +- .../network/peers/utils/enrSubnets.test.ts | 2 +- .../test/unit/network/reqresp/utils.ts | 2 +- .../test/unit/network/util.test.ts | 2 +- .../test/unit/sync/backfill/verify.test.ts | 2 +- .../test/unit/sync/unknownBlock.test.ts | 4 +- .../unit/sync/utils/remoteSyncType.test.ts | 2 +- packages/beacon-node/test/utils/clock.ts | 5 +- packages/beacon-node/test/utils/network.ts | 2 +- .../beacon-node/test/utils/node/simTest.ts | 2 +- .../beacon-node/test/utils/node/validator.ts | 2 +- packages/beacon-node/test/utils/state.ts | 2 +- packages/beacon-node/test/utils/testnet.ts | 2 +- .../utils/validationData/aggregateAndProof.ts | 3 +- .../test/utils/validationData/attestation.ts | 9 +- packages/cli/package.json | 3 + .../cli/src/cmds/beacon/initPeerIdAndEnr.ts | 2 +- packages/cli/src/cmds/dev/files.ts | 2 +- packages/cli/src/cmds/lightclient/handler.ts | 2 +- .../cmds/validator/blsToExecutionChange.ts | 6 +- .../keymanager/decryptKeystoreDefinitions.ts | 2 +- .../keymanager/decryptKeystores/threadPool.ts | 5 +- .../cli/src/cmds/validator/keymanager/impl.ts | 2 +- .../validator/keymanager/keystoreCache.ts | 2 +- .../src/cmds/validator/keymanager/server.ts | 2 +- .../cli/src/cmds/validator/signers/index.ts | 2 +- packages/cli/src/config/peerId.ts | 2 +- .../cli/test/e2e/blsToExecutionchange.test.ts | 2 +- packages/cli/test/unit/cmds/beacon.test.ts | 2 +- .../keymanager/keystoreCache.test.ts | 2 +- packages/cli/test/unit/db.test.ts | 2 + .../cli/test/utils/mockBeaconApiServer.ts | 1 + .../simulation/assertions/nodeAssertion.ts | 2 +- .../cli/test/utils/simulation/interfaces.ts | 2 +- .../cli/test/utils/simulation/utils/index.ts | 11 +- packages/config/src/genesisConfig/index.ts | 6 +- packages/config/test/unit/index.test.ts | 2 +- packages/db/src/abstractRepository.ts | 2 +- packages/flare/package.json | 4 + packages/fork-choice/src/forkChoice/store.ts | 2 +- .../fork-choice/src/protoArray/protoArray.ts | 2 +- .../perf/forkChoice/onAttestation.test.ts | 2 +- .../fork-choice/test/perf/forkChoice/util.ts | 2 +- .../perf/protoArray/computeDeltas.test.ts | 1 + .../test/unit/forkChoice/forkChoice.test.ts | 2 +- packages/light-client/src/index.ts | 2 +- .../src/spec/validateLightClientUpdate.ts | 4 +- packages/light-client/src/utils/domain.ts | 2 +- packages/light-client/src/utils/utils.ts | 2 +- packages/light-client/src/validation.ts | 2 +- .../unit/isValidLightClientHeader.test.ts | 2 +- .../light-client/test/unit/sync.node.test.ts | 4 +- .../test/utils/prepareUpdateNaive.ts | 4 +- packages/logger/src/node.ts | 5 +- packages/logger/test/unit/logger.test.ts | 1 + packages/prover/package.json | 3 +- .../eth_getBlockByHash.test.ts | 2 +- .../eth_getBlockByNumber.test.ts | 2 +- packages/reqresp/package.json | 1 + packages/reqresp/src/ReqResp.ts | 7 +- .../reqresp/src/encoders/requestDecode.ts | 2 +- packages/reqresp/src/request/index.ts | 2 +- packages/reqresp/test/fixtures/messages.ts | 2 +- packages/reqresp/test/fixtures/protocols.ts | 2 +- .../src/block/initiateValidatorExit.ts | 2 +- .../src/block/processAttestationPhase0.ts | 2 +- .../src/block/processAttestationsAltair.ts | 2 +- .../src/block/processBlsToExecutionChange.ts | 4 +- .../src/block/processEth1Data.ts | 4 +- .../src/block/processExecutionPayload.ts | 2 +- .../src/block/processSyncCommittee.ts | 2 +- .../src/block/processWithdrawals.ts | 2 +- .../src/cache/syncCommitteeCache.ts | 2 +- packages/state-transition/src/cache/types.ts | 2 +- .../processJustificationAndFinalization.ts | 2 +- .../src/epoch/processPendingAttestations.ts | 2 +- .../src/signatureSets/blsToExecutionChange.ts | 4 +- packages/state-transition/src/slot/index.ts | 2 +- .../src/slot/upgradeStateToAltair.ts | 2 +- .../state-transition/src/stateTransition.ts | 2 +- packages/state-transition/src/util/genesis.ts | 2 +- .../state-transition/src/util/signingRoot.ts | 2 +- .../src/util/weakSubjectivity.ts | 2 +- .../state-transition/test/perf/block/util.ts | 2 +- .../test/perf/misc/aggregationBits.test.ts | 2 +- .../test/perf/misc/rootEquals.test.ts | 2 +- packages/state-transition/test/perf/util.ts | 4 +- .../unit/signatureSets/signatureSets.test.ts | 2 +- .../test/unit/util/seed.test.ts | 2 +- packages/test-utils/package.json | 2 +- packages/utils/src/map.ts | 5 +- packages/validator/src/index.ts | 2 - .../validator/src/services/attestation.ts | 2 +- .../src/services/attestationDuties.ts | 2 +- packages/validator/src/services/block.ts | 2 +- .../validator/src/services/blockDuties.ts | 2 +- .../src/services/chainHeaderTracker.ts | 2 +- packages/validator/src/services/indices.ts | 8 +- .../src/services/syncCommitteeDuties.ts | 2 +- .../validator/src/services/validatorStore.ts | 4 +- .../attestationByTargetRepository.ts | 2 +- .../attestationLowerBoundRepository.ts | 2 +- .../block/blockBySlotRepository.ts | 2 +- .../validator/src/slashingProtection/index.ts | 2 +- .../minMaxSurround/distanceStoreRepository.ts | 7 +- .../validator/src/slashingProtection/utils.ts | 2 +- packages/validator/src/util/difference.ts | 2 +- .../src/util/externalSignerClient.ts | 2 +- packages/validator/src/validator.ts | 8 +- .../validator/test/e2e/web3signer.test.ts | 2 +- .../unit/services/attestationDuties.test.ts | 2 +- .../interchange/index.test.ts | 2 +- .../test/unit/validatorStore.test.ts | 2 +- packages/validator/test/utils/spec.ts | 2 +- yarn.lock | 465 +++++++++++++----- 244 files changed, 739 insertions(+), 410 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index b02fd4fbf1d6..411eba578366 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -97,7 +97,11 @@ module.exports = { "import/no-duplicates": "off", "import/no-extraneous-dependencies": [ "error", - {devDependencies: false, optionalDependencies: false, peerDependencies: false}, + { + devDependencies: false, + optionalDependencies: false, + peerDependencies: false, + }, ], "import/no-relative-packages": "error", // TEMP Disabled while eslint-plugin-import support ESM (Typescript does support it) https://github.com/import-js/eslint-plugin-import/issues/2170 @@ -152,7 +156,6 @@ module.exports = { semi: "off", }, settings: { - "import/internal-regex": "^@chainsafe/", "import/core-modules": [ "node:child_process", "node:crypto", @@ -165,6 +168,11 @@ module.exports = { "node:util", "node:url", ], + "import/resolver": { + typescript: { + project: "packages/*/tsconfig.json", + }, + }, }, overrides: [ { diff --git a/package.json b/package.json index 10a8c70bb391..b4f7611a33c5 100644 --- a/package.json +++ b/package.json @@ -44,18 +44,19 @@ "@types/node": "^18.15.11", "@types/sinon": "^10.0.13", "@types/sinon-chai": "^3.2.9", - "@typescript-eslint/eslint-plugin": "5.60.1", - "@typescript-eslint/parser": "5.60.1", + "@typescript-eslint/eslint-plugin": "6.0.0", + "@typescript-eslint/parser": "6.0.0", "chai": "^4.3.7", "chai-as-promised": "^7.1.1", "codecov": "^3.8.3", "crypto-browserify": "^3.12.0", "electron": "^21.0.1", - "eslint": "^8.43.0", + "eslint": "^8.44.0", "eslint-plugin-import": "^2.27.5", - "eslint-plugin-prettier": "^4.2.1", + "eslint-plugin-prettier": "^5.0.0", "eslint-plugin-chai-expect": "^3.0.0", "eslint-plugin-mocha": "^10.1.0", + "eslint-import-resolver-typescript": "^3.5.5", "https-browserify": "^1.0.0", "karma": "^6.4.1", "karma-chai": "^0.1.0", @@ -73,7 +74,7 @@ "npm-run-all": "^4.1.5", "nyc": "^15.1.0", "path-browserify": "^1.0.1", - "prettier": "^2.8.8", + "prettier": "^3.0.0", "process": "^0.11.10", "resolve-typescript-plugin": "^2.0.1", "sinon": "^15.0.3", diff --git a/packages/api/src/beacon/client/proof.ts b/packages/api/src/beacon/client/proof.ts index 0c37caaad954..5188725da148 100644 --- a/packages/api/src/beacon/client/proof.ts +++ b/packages/api/src/beacon/client/proof.ts @@ -1,5 +1,5 @@ -import {ChainForkConfig} from "@lodestar/config"; import {CompactMultiProof, ProofType} from "@chainsafe/persistent-merkle-tree"; +import {ChainForkConfig} from "@lodestar/config"; import {Api, ReqTypes, routesData, getReqSerializers} from "../routes/proof.js"; import {IHttpClient, getFetchOptsSerializers, HttpError} from "../../utils/client/index.js"; import {HttpStatusCode} from "../../utils/client/httpStatusCode.js"; diff --git a/packages/api/src/beacon/routes/config.ts b/packages/api/src/beacon/routes/config.ts index 89ef68e051dc..0d009c844fa2 100644 --- a/packages/api/src/beacon/routes/config.ts +++ b/packages/api/src/beacon/routes/config.ts @@ -1,8 +1,8 @@ +import {ByteVectorType, ContainerType} from "@chainsafe/ssz"; import {BeaconPreset} from "@lodestar/params"; import {ChainConfig} from "@lodestar/config"; import {Bytes32, UintNum64, phase0, ssz} from "@lodestar/types"; import {mapValues} from "@lodestar/utils"; -import {ByteVectorType, ContainerType} from "@chainsafe/ssz"; import { ArrayOf, ReqEmpty, diff --git a/packages/api/src/beacon/routes/debug.ts b/packages/api/src/beacon/routes/debug.ts index 0be8ec38a255..419a785e84de 100644 --- a/packages/api/src/beacon/routes/debug.ts +++ b/packages/api/src/beacon/routes/debug.ts @@ -1,6 +1,6 @@ +import {ContainerType, ValueOf} from "@chainsafe/ssz"; import {ForkName} from "@lodestar/params"; import {allForks, Slot, RootHex, ssz, StringType} from "@lodestar/types"; -import {ContainerType, ValueOf} from "@chainsafe/ssz"; import { ArrayOf, ReturnTypes, diff --git a/packages/api/src/beacon/routes/events.ts b/packages/api/src/beacon/routes/events.ts index 3d25f677205d..7e94ab686f22 100644 --- a/packages/api/src/beacon/routes/events.ts +++ b/packages/api/src/beacon/routes/events.ts @@ -1,5 +1,5 @@ -import {Epoch, phase0, capella, Slot, ssz, StringType, RootHex, altair, UintNum64, allForks} from "@lodestar/types"; import {ContainerType} from "@chainsafe/ssz"; +import {Epoch, phase0, capella, Slot, ssz, StringType, RootHex, altair, UintNum64, allForks} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {isForkExecution, ForkName} from "@lodestar/params"; diff --git a/packages/api/src/beacon/routes/node.ts b/packages/api/src/beacon/routes/node.ts index e54c08635040..a9476e7e592b 100644 --- a/packages/api/src/beacon/routes/node.ts +++ b/packages/api/src/beacon/routes/node.ts @@ -1,5 +1,5 @@ -import {allForks, ssz, StringType} from "@lodestar/types"; import {ContainerType} from "@chainsafe/ssz"; +import {allForks, ssz, StringType} from "@lodestar/types"; import { ArrayOf, reqEmpty, diff --git a/packages/api/src/beacon/server/proof.ts b/packages/api/src/beacon/server/proof.ts index cc54249cf447..a03ae472bf78 100644 --- a/packages/api/src/beacon/server/proof.ts +++ b/packages/api/src/beacon/server/proof.ts @@ -1,5 +1,5 @@ -import {ChainForkConfig} from "@lodestar/config"; import {CompactMultiProof} from "@chainsafe/persistent-merkle-tree"; +import {ChainForkConfig} from "@lodestar/config"; import {Api, ReqTypes, routesData, getReturnTypes, getReqSerializers} from "../routes/proof.js"; import {ServerRoutes, getGenericJsonServer} from "../../utils/server/index.js"; import {ServerApi} from "../../interfaces.js"; diff --git a/packages/api/src/builder/routes.ts b/packages/api/src/builder/routes.ts index 520f5699c337..b3bc3bb38efc 100644 --- a/packages/api/src/builder/routes.ts +++ b/packages/api/src/builder/routes.ts @@ -1,5 +1,5 @@ -import {ssz, allForks, bellatrix, Slot, Root, BLSPubkey} from "@lodestar/types"; import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {ssz, allForks, bellatrix, Slot, Root, BLSPubkey} from "@lodestar/types"; import {ForkName, isForkExecution} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; diff --git a/packages/api/src/interfaces.ts b/packages/api/src/interfaces.ts index b9e3c3a92d59..b31d6ddfc10e 100644 --- a/packages/api/src/interfaces.ts +++ b/packages/api/src/interfaces.ts @@ -15,7 +15,7 @@ export type ApiClientSErrorResponse = {[K in HttpSuccessCodes]: unknown}, - E extends Exclude = Exclude + E extends Exclude = Exclude, > = | {[K in keyof S]: ApiClientSuccessResponse}[keyof S] | {[K in E]: ApiClientSErrorResponse}[E] diff --git a/packages/api/src/utils/client/client.ts b/packages/api/src/utils/client/client.ts index 8fb8374d60b4..55b5e20f628e 100644 --- a/packages/api/src/utils/client/client.ts +++ b/packages/api/src/utils/client/client.ts @@ -40,7 +40,7 @@ export function getFetchOptsSerializer any, ReqType // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getFetchOptsSerializers< Api extends Record, - ReqTypes extends {[K in keyof Api]: ReqGeneric} + ReqTypes extends {[K in keyof Api]: ReqGeneric}, >(routesData: RoutesData, reqSerializers: ReqSerializers) { return mapValues(routesData, (routeDef, routeId) => getFetchOptsSerializer(routeDef, reqSerializers[routeId], routeId as string) @@ -52,7 +52,7 @@ export function getFetchOptsSerializers< */ export function generateGenericJsonClient< Api extends Record, - ReqTypes extends {[K in keyof Api]: ReqGeneric} + ReqTypes extends {[K in keyof Api]: ReqGeneric}, >( routesData: RoutesData, reqSerializers: ReqSerializers, diff --git a/packages/api/src/utils/schema.ts b/packages/api/src/utils/schema.ts index 512a18ea0f8b..03af48195f86 100644 --- a/packages/api/src/utils/schema.ts +++ b/packages/api/src/utils/schema.ts @@ -86,7 +86,7 @@ function isRequired(schema: Schema): boolean { export function getFastifySchema(schemaDef: SchemaDefinition): JsonSchema { const schema: {params?: JsonSchemaObj; querystring?: JsonSchemaObj; body?: JsonSchema} = {}; - if (schemaDef.body) { + if (schemaDef.body != null) { schema.body = getJsonSchemaItem(schemaDef.body); } diff --git a/packages/api/src/utils/server/genericJsonServer.ts b/packages/api/src/utils/server/genericJsonServer.ts index fa0d115982fe..ebda05ccc859 100644 --- a/packages/api/src/utils/server/genericJsonServer.ts +++ b/packages/api/src/utils/server/genericJsonServer.ts @@ -13,14 +13,14 @@ import {ServerRoute} from "./types.js"; export type ServerRoutes< Api extends Record, - ReqTypes extends {[K in keyof Api]: ReqGeneric} + ReqTypes extends {[K in keyof Api]: ReqGeneric}, > = { [K in keyof Api]: ServerRoute; }; export function getGenericJsonServer< Api extends Record, - ReqTypes extends {[K in keyof Api]: ReqGeneric} + ReqTypes extends {[K in keyof Api]: ReqGeneric}, >( {routesData, getReqSerializers, getReturnTypes}: RouteGroupDefinition, config: ChainForkConfig, diff --git a/packages/api/src/utils/types.ts b/packages/api/src/utils/types.ts index f31d7a0bd0dd..a62fb30b7270 100644 --- a/packages/api/src/utils/types.ts +++ b/packages/api/src/utils/types.ts @@ -14,7 +14,7 @@ const codeCase = "camel" as const; export type RouteGroupDefinition< Api extends Record, - ReqTypes extends {[K in keyof Api]: ReqGeneric} + ReqTypes extends {[K in keyof Api]: ReqGeneric}, > = { routesData: RoutesData; getReqSerializers: (config: ChainForkConfig) => ReqSerializers; @@ -54,7 +54,7 @@ export type ReqSerializer any, ReqType extends ReqG export type ReqSerializers< Api extends Record, - ReqTypes extends {[K in keyof Api]: ReqGeneric} + ReqTypes extends {[K in keyof Api]: ReqGeneric}, > = { [K in keyof Api]: ReqSerializer; }; diff --git a/packages/api/test/unit/beacon/genericServerTest/debug.test.ts b/packages/api/test/unit/beacon/genericServerTest/debug.test.ts index df91d5e081e2..44b080e29bf4 100644 --- a/packages/api/test/unit/beacon/genericServerTest/debug.test.ts +++ b/packages/api/test/unit/beacon/genericServerTest/debug.test.ts @@ -1,6 +1,6 @@ import {expect} from "chai"; -import {ssz} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {ssz} from "@lodestar/types"; import {config} from "@lodestar/config/default"; import {Api, ReqTypes, routesData} from "../../../../src/beacon/routes/debug.js"; import {getClient} from "../../../../src/beacon/client/debug.js"; diff --git a/packages/api/test/unit/beacon/testData/beacon.ts b/packages/api/test/unit/beacon/testData/beacon.ts index 634455092ede..2cca22564431 100644 --- a/packages/api/test/unit/beacon/testData/beacon.ts +++ b/packages/api/test/unit/beacon/testData/beacon.ts @@ -1,6 +1,6 @@ +import {toHexString} from "@chainsafe/ssz"; import {ForkName} from "@lodestar/params"; import {ssz, Slot, allForks} from "@lodestar/types"; -import {toHexString} from "@chainsafe/ssz"; import {Api, BlockHeaderResponse, ValidatorResponse} from "../../../../src/beacon/routes/beacon/index.js"; import {GenericServerTestCases} from "../../../utils/genericServerTest.js"; diff --git a/packages/api/test/unit/beacon/testData/debug.ts b/packages/api/test/unit/beacon/testData/debug.ts index 5b772098066b..6b65d610d16f 100644 --- a/packages/api/test/unit/beacon/testData/debug.ts +++ b/packages/api/test/unit/beacon/testData/debug.ts @@ -1,6 +1,6 @@ +import {toHexString} from "@chainsafe/ssz"; import {ForkName} from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {toHexString} from "@chainsafe/ssz"; import {Api} from "../../../../src/beacon/routes/debug.js"; import {GenericServerTestCases} from "../../../utils/genericServerTest.js"; diff --git a/packages/api/test/unit/beacon/testData/lightclient.ts b/packages/api/test/unit/beacon/testData/lightclient.ts index 36947483075c..553f11d685d1 100644 --- a/packages/api/test/unit/beacon/testData/lightclient.ts +++ b/packages/api/test/unit/beacon/testData/lightclient.ts @@ -1,5 +1,5 @@ -import {ssz} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {ssz} from "@lodestar/types"; import {ForkName} from "@lodestar/params"; import {Api} from "../../../../src/beacon/routes/lightclient.js"; import {GenericServerTestCases} from "../../../utils/genericServerTest.js"; diff --git a/packages/api/test/unit/builder/testData.ts b/packages/api/test/unit/builder/testData.ts index dc7a28fc9d2b..94ef3c393b20 100644 --- a/packages/api/test/unit/builder/testData.ts +++ b/packages/api/test/unit/builder/testData.ts @@ -1,5 +1,5 @@ -import {ssz} from "@lodestar/types"; import {fromHexString} from "@chainsafe/ssz"; +import {ssz} from "@lodestar/types"; import {ForkName} from "@lodestar/params"; import {Api} from "../../../src/builder/routes.js"; diff --git a/packages/api/test/utils/genericServerTest.ts b/packages/api/test/utils/genericServerTest.ts index cfe89be15c53..13a1860087e4 100644 --- a/packages/api/test/utils/genericServerTest.ts +++ b/packages/api/test/utils/genericServerTest.ts @@ -20,7 +20,7 @@ export type GenericServerTestCases> export function runGenericServerTest< Api extends Record, - ReqTypes extends {[K in keyof Api]: ReqGeneric} + ReqTypes extends {[K in keyof Api]: ReqGeneric}, >( config: ChainForkConfig, getClient: (config: ChainForkConfig, https: IHttpClient) => Api, diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index b9b775f72544..753ab35ca5e6 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -116,6 +116,7 @@ "@libp2p/interface-registrar": "^2.0.12", "@libp2p/mdns": "^8.0.0", "@libp2p/mplex": "^8.0.3", + "@libp2p/peer-id": "^2.0.3", "@libp2p/peer-id-factory": "^2.0.3", "@libp2p/prometheus-metrics": "^1.1.4", "@libp2p/tcp": "7.0.1", @@ -124,6 +125,7 @@ "@lodestar/db": "^1.9.1", "@lodestar/fork-choice": "^1.9.1", "@lodestar/light-client": "^1.9.1", + "@lodestar/logger": "^1.9.1", "@lodestar/params": "^1.9.1", "@lodestar/reqresp": "^1.9.1", "@lodestar/state-transition": "^1.9.1", @@ -144,6 +146,7 @@ "it-pipe": "^3.0.1", "jwt-simple": "0.5.6", "libp2p": "0.45.9", + "multiformats": "^11.0.1", "prom-client": "^14.2.0", "qs": "^6.11.1", "snappyjs": "^0.7.0", @@ -155,7 +158,6 @@ "xxhash-wasm": "1.0.2" }, "devDependencies": { - "@lodestar/logger": "^1.9.1", "@types/eventsource": "^1.1.11", "@types/leveldown": "^4.0.3", "@types/qs": "^6.9.7", diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 7833775b4c9a..663e316f413c 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -1,9 +1,9 @@ +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {routes, ServerApi, isSignedBlockContents, isSignedBlindedBlockContents} from "@lodestar/api"; import {computeTimeAtSlot} from "@lodestar/state-transition"; import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {sleep} from "@lodestar/utils"; import {allForks, deneb} from "@lodestar/types"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import { BlockSource, getBlockInput, diff --git a/packages/beacon-node/src/api/impl/beacon/state/utils.ts b/packages/beacon-node/src/api/impl/beacon/state/utils.ts index fda6472a2a82..3c17fa30ac67 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/utils.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/utils.ts @@ -1,9 +1,9 @@ +import {fromHexString} from "@chainsafe/ssz"; import {routes} from "@lodestar/api"; import {FAR_FUTURE_EPOCH, GENESIS_SLOT} from "@lodestar/params"; import {BeaconStateAllForks, PubkeyIndexMap} from "@lodestar/state-transition"; import {BLSPubkey, phase0} from "@lodestar/types"; import {Epoch, ValidatorIndex} from "@lodestar/types"; -import {fromHexString} from "@chainsafe/ssz"; import {IBeaconChain, StateGetOpts} from "../../../../chain/index.js"; import {ApiError, ValidationError} from "../../errors.js"; import {isOptimisticBlock} from "../../../../util/forkChoice.js"; diff --git a/packages/beacon-node/src/api/impl/lightclient/index.ts b/packages/beacon-node/src/api/impl/lightclient/index.ts index 52c6ca33c41d..83052630f343 100644 --- a/packages/beacon-node/src/api/impl/lightclient/index.ts +++ b/packages/beacon-node/src/api/impl/lightclient/index.ts @@ -1,5 +1,5 @@ -import {routes, ServerApi} from "@lodestar/api"; import {fromHexString} from "@chainsafe/ssz"; +import {routes, ServerApi} from "@lodestar/api"; import {SyncPeriod} from "@lodestar/types"; import {MAX_REQUEST_LIGHT_CLIENT_UPDATES, MAX_REQUEST_LIGHT_CLIENT_COMMITTEE_HASHES} from "@lodestar/params"; import {ApiModules} from "../types.js"; diff --git a/packages/beacon-node/src/api/impl/lodestar/index.ts b/packages/beacon-node/src/api/impl/lodestar/index.ts index 278c7e4112a0..62e336138191 100644 --- a/packages/beacon-node/src/api/impl/lodestar/index.ts +++ b/packages/beacon-node/src/api/impl/lodestar/index.ts @@ -1,8 +1,8 @@ +import {toHexString} from "@chainsafe/ssz"; import {routes, ServerApi} from "@lodestar/api"; import {Repository} from "@lodestar/db"; import {toHex} from "@lodestar/utils"; import {getLatestWeakSubjectivityCheckpointEpoch} from "@lodestar/state-transition"; -import {toHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {ssz} from "@lodestar/types"; import {BeaconChain} from "../../../chain/index.js"; diff --git a/packages/beacon-node/src/api/impl/proof/index.ts b/packages/beacon-node/src/api/impl/proof/index.ts index 06fdc9c4b246..09f691c4c453 100644 --- a/packages/beacon-node/src/api/impl/proof/index.ts +++ b/packages/beacon-node/src/api/impl/proof/index.ts @@ -1,5 +1,5 @@ -import {routes, ServerApi} from "@lodestar/api"; import {createProof, ProofType} from "@chainsafe/persistent-merkle-tree"; +import {routes, ServerApi} from "@lodestar/api"; import {ApiModules} from "../types.js"; import {resolveStateId} from "../beacon/state/utils.js"; import {resolveBlockId} from "../beacon/blocks/utils.js"; diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index def416fc963a..42e3783fa920 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -1,3 +1,4 @@ +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {routes, ServerApi, BlockContents} from "@lodestar/api"; import { CachedBeaconStateAllForks, @@ -18,7 +19,6 @@ import { import {Root, Slot, ValidatorIndex, ssz, Epoch, ProducedBlockSource, bellatrix, allForks} from "@lodestar/types"; import {ExecutionStatus} from "@lodestar/fork-choice"; import {toHex} from "@lodestar/utils"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import {AttestationError, AttestationErrorCode, GossipAction, SyncCommitteeError} from "../../../chain/errors/index.js"; import {validateGossipAggregateAndProof} from "../../../chain/validation/index.js"; import {ZERO_HASH} from "../../../constants/index.js"; diff --git a/packages/beacon-node/src/api/rest/base.ts b/packages/beacon-node/src/api/rest/base.ts index 57227be7956b..fda16fffc8ce 100644 --- a/packages/beacon-node/src/api/rest/base.ts +++ b/packages/beacon-node/src/api/rest/base.ts @@ -44,7 +44,10 @@ export class RestApiServer { private status = Status.Closed; - constructor(private readonly opts: RestApiServerOpts, modules: RestApiServerModules) { + constructor( + private readonly opts: RestApiServerOpts, + modules: RestApiServerModules + ) { // Apply opts defaults const {logger, metrics} = modules; diff --git a/packages/beacon-node/src/chain/beaconProposerCache.ts b/packages/beacon-node/src/chain/beaconProposerCache.ts index 59040344f464..c6149b0232c9 100644 --- a/packages/beacon-node/src/chain/beaconProposerCache.ts +++ b/packages/beacon-node/src/chain/beaconProposerCache.ts @@ -9,7 +9,10 @@ export type ProposerPreparationData = routes.validator.ProposerPreparationData; export class BeaconProposerCache { private readonly feeRecipientByValidatorIndex: MapDef; - constructor(opts: {suggestedFeeRecipient: string}, private readonly metrics?: Metrics | null) { + constructor( + opts: {suggestedFeeRecipient: string}, + private readonly metrics?: Metrics | null + ) { this.feeRecipientByValidatorIndex = new MapDef(() => ({ epoch: 0, feeRecipient: opts.suggestedFeeRecipient, diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 8da566b9d2bb..63cabc225290 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -1,6 +1,6 @@ +import {toHexString} from "@chainsafe/ssz"; import {capella, ssz, allForks, altair} from "@lodestar/types"; import {ForkSeq, INTERVALS_PER_SLOT, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params"; -import {toHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAltair, computeEpochAtSlot, diff --git a/packages/beacon-node/src/chain/blocks/verifyBlock.ts b/packages/beacon-node/src/chain/blocks/verifyBlock.ts index 2435c0cc47e4..77c6b4832bbc 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlock.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlock.ts @@ -1,3 +1,4 @@ +import {toHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAllForks, computeEpochAtSlot, @@ -6,7 +7,6 @@ import { } from "@lodestar/state-transition"; import {bellatrix} from "@lodestar/types"; import {ForkName} from "@lodestar/params"; -import {toHexString} from "@chainsafe/ssz"; import {ProtoBlock} from "@lodestar/fork-choice"; import {ChainForkConfig} from "@lodestar/config"; import {Logger} from "@lodestar/utils"; diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts index ccfc917421b2..6925dd543d80 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts @@ -1,3 +1,4 @@ +import {toHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAllForks, isExecutionStateType, @@ -7,7 +8,6 @@ import { kzgCommitmentToVersionedHash, } from "@lodestar/state-transition"; import {bellatrix, allForks, Slot, deneb} from "@lodestar/types"; -import {toHexString} from "@chainsafe/ssz"; import { IForkChoice, assertValidTerminalPowBlock, @@ -21,7 +21,7 @@ import {ChainForkConfig} from "@lodestar/config"; import {ErrorAborted, Logger} from "@lodestar/utils"; import {ForkSeq} from "@lodestar/params"; -import {IExecutionEngine} from "../../execution/engine/index.js"; +import {IExecutionEngine} from "../../execution/engine/interface.js"; import {BlockError, BlockErrorCode} from "../errors/index.js"; import {IClock} from "../../util/clock.js"; import {BlockProcessOpts} from "../options.js"; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index ccb2c023c5fb..9e91b329b0d4 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import {CompositeTypeAny, fromHexString, TreeView, Type} from "@chainsafe/ssz"; import { BeaconStateAllForks, CachedBeaconStateAllForks, @@ -28,7 +29,6 @@ import { import {CheckpointWithHex, ExecutionStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {ProcessShutdownCallback} from "@lodestar/validator"; import {Logger, isErrorAborted, pruneSetToMax, sleep, toHex} from "@lodestar/utils"; -import {CompositeTypeAny, fromHexString, TreeView, Type} from "@chainsafe/ssz"; import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js"; diff --git a/packages/beacon-node/src/chain/errors/attestationError.ts b/packages/beacon-node/src/chain/errors/attestationError.ts index b2b573d99d1b..fa380d048c7e 100644 --- a/packages/beacon-node/src/chain/errors/attestationError.ts +++ b/packages/beacon-node/src/chain/errors/attestationError.ts @@ -1,5 +1,5 @@ -import {CommitteeIndex, Epoch, Slot, ValidatorIndex, RootHex} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {CommitteeIndex, Epoch, Slot, ValidatorIndex, RootHex} from "@lodestar/types"; import {GossipActionError} from "./gossipValidation.js"; export enum AttestationErrorCode { diff --git a/packages/beacon-node/src/chain/errors/blockError.ts b/packages/beacon-node/src/chain/errors/blockError.ts index d9d42fdbc221..7438e662614b 100644 --- a/packages/beacon-node/src/chain/errors/blockError.ts +++ b/packages/beacon-node/src/chain/errors/blockError.ts @@ -1,6 +1,6 @@ +import {toHexString} from "@chainsafe/ssz"; import {allForks, RootHex, Slot, ValidatorIndex} from "@lodestar/types"; import {LodestarError} from "@lodestar/utils"; -import {toHexString} from "@chainsafe/ssz"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {ExecutePayloadStatus} from "../../execution/engine/interface.js"; import {QueueErrorCode} from "../../util/queue/index.js"; @@ -108,7 +108,10 @@ export type BlockErrorType = export class BlockGossipError extends GossipActionError {} export class BlockError extends LodestarError { - constructor(readonly signedBlock: allForks.SignedBeaconBlock, type: BlockErrorType) { + constructor( + readonly signedBlock: allForks.SignedBeaconBlock, + type: BlockErrorType + ) { super(type); } diff --git a/packages/beacon-node/src/chain/genesis/genesis.ts b/packages/beacon-node/src/chain/genesis/genesis.ts index 861f1fa70efb..979476c69530 100644 --- a/packages/beacon-node/src/chain/genesis/genesis.ts +++ b/packages/beacon-node/src/chain/genesis/genesis.ts @@ -1,7 +1,7 @@ +import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; import {GENESIS_EPOCH, GENESIS_SLOT} from "@lodestar/params"; import {phase0, ssz} from "@lodestar/types"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; -import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; import { getTemporaryBlockHeader, getGenesisBeaconState, diff --git a/packages/beacon-node/src/chain/genesis/interface.ts b/packages/beacon-node/src/chain/genesis/interface.ts index 32c19207ba2f..d1bae335b9ff 100644 --- a/packages/beacon-node/src/chain/genesis/interface.ts +++ b/packages/beacon-node/src/chain/genesis/interface.ts @@ -1,6 +1,6 @@ +import {CompositeViewDU, VectorCompositeType} from "@chainsafe/ssz"; import {ssz} from "@lodestar/types"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; -import {CompositeViewDU, VectorCompositeType} from "@chainsafe/ssz"; import {Eth1Block} from "../../eth1/interface.js"; export type GenesisResult = { diff --git a/packages/beacon-node/src/chain/initState.ts b/packages/beacon-node/src/chain/initState.ts index 40ace9c43ef8..a0bd5c4968a1 100644 --- a/packages/beacon-node/src/chain/initState.ts +++ b/packages/beacon-node/src/chain/initState.ts @@ -1,3 +1,4 @@ +import {toHexString} from "@chainsafe/ssz"; import { blockToHeader, computeEpochAtSlot, @@ -9,7 +10,6 @@ import { import {phase0, allForks, ssz} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {Logger, toHex} from "@lodestar/utils"; -import {toHexString} from "@chainsafe/ssz"; import {GENESIS_SLOT, ZERO_HASH} from "../constants/index.js"; import {IBeaconDb} from "../db/index.js"; import {Eth1Provider} from "../eth1/index.js"; diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 5f80c1411220..1edd646af83d 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -1,3 +1,4 @@ +import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {allForks, UintNum64, Root, phase0, Slot, RootHex, Epoch, ValidatorIndex, deneb, Wei} from "@lodestar/types"; import { BeaconStateAllForks, @@ -6,7 +7,6 @@ import { PubkeyIndexMap, } from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; -import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {Logger} from "@lodestar/utils"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; diff --git a/packages/beacon-node/src/chain/lightClient/index.ts b/packages/beacon-node/src/chain/lightClient/index.ts index d1460c138c4f..202d1adc3e3d 100644 --- a/packages/beacon-node/src/chain/lightClient/index.ts +++ b/packages/beacon-node/src/chain/lightClient/index.ts @@ -1,3 +1,4 @@ +import {BitArray, CompositeViewDU, toHexString} from "@chainsafe/ssz"; import {altair, phase0, Root, RootHex, Slot, ssz, SyncPeriod, allForks} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import { @@ -15,7 +16,6 @@ import { } from "@lodestar/light-client/spec"; import {Logger, MapDef, pruneSetToMax} from "@lodestar/utils"; import {routes} from "@lodestar/api"; -import {BitArray, CompositeViewDU, toHexString} from "@chainsafe/ssz"; import {MIN_SYNC_COMMITTEE_PARTICIPANTS, SYNC_COMMITTEE_SIZE, ForkName, ForkSeq, ForkExecution} from "@lodestar/params"; import {IBeaconDb} from "../../db/index.js"; @@ -184,7 +184,10 @@ export class LightClientServer { private readonly zero: Pick; private finalized: allForks.LightClientFinalityUpdate | null = null; - constructor(private readonly opts: LightClientServerOpts, modules: LightClientServerModules) { + constructor( + private readonly opts: LightClientServerOpts, + modules: LightClientServerModules + ) { const {config, db, metrics, emitter, logger} = modules; this.config = config; this.db = db; diff --git a/packages/beacon-node/src/chain/lightClient/proofs.ts b/packages/beacon-node/src/chain/lightClient/proofs.ts index 6f68e0eae7c4..cf1735e706d6 100644 --- a/packages/beacon-node/src/chain/lightClient/proofs.ts +++ b/packages/beacon-node/src/chain/lightClient/proofs.ts @@ -1,7 +1,7 @@ +import {Tree} from "@chainsafe/persistent-merkle-tree"; import {BeaconStateAllForks} from "@lodestar/state-transition"; import {FINALIZED_ROOT_GINDEX, BLOCK_BODY_EXECUTION_PAYLOAD_GINDEX, ForkExecution} from "@lodestar/params"; import {allForks, ssz} from "@lodestar/types"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; import {SyncCommitteeWitness} from "./types.js"; diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index 5432fc62cfd7..e531e2c39cf6 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -1,4 +1,5 @@ import bls from "@chainsafe/bls"; +import {toHexString} from "@chainsafe/ssz"; import { ForkName, MAX_ATTESTATIONS, @@ -15,7 +16,6 @@ import { computeStartSlotAtEpoch, getBlockRootAtSlot, } from "@lodestar/state-transition"; -import {toHexString} from "@chainsafe/ssz"; import {IForkChoice, EpochDifference} from "@lodestar/fork-choice"; import {toHex, MapDef} from "@lodestar/utils"; import {intersectUint8Arrays, IntersectResult} from "../../util/bitArray.js"; @@ -217,7 +217,10 @@ type AttestationNonParticipant = { export class MatchingDataAttestationGroup { private readonly attestations: AttestationWithIndex[] = []; - constructor(readonly committee: ValidatorIndex[], readonly data: phase0.AttestationData) {} + constructor( + readonly committee: ValidatorIndex[], + readonly data: phase0.AttestationData + ) {} getAttestationCount(): number { return this.attestations.length; diff --git a/packages/beacon-node/src/chain/opPools/attestationPool.ts b/packages/beacon-node/src/chain/opPools/attestationPool.ts index 175394125264..a04cd7592091 100644 --- a/packages/beacon-node/src/chain/opPools/attestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/attestationPool.ts @@ -1,7 +1,7 @@ -import {phase0, Slot, Root, RootHex} from "@lodestar/types"; import {PointFormat, Signature} from "@chainsafe/bls/types"; import bls from "@chainsafe/bls"; import {BitArray, toHexString} from "@chainsafe/ssz"; +import {phase0, Slot, Root, RootHex} from "@lodestar/types"; import {MapDef} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js"; diff --git a/packages/beacon-node/src/chain/opPools/opPool.ts b/packages/beacon-node/src/chain/opPools/opPool.ts index 67e68fd8bee2..b2a49ae4c07c 100644 --- a/packages/beacon-node/src/chain/opPools/opPool.ts +++ b/packages/beacon-node/src/chain/opPools/opPool.ts @@ -1,3 +1,4 @@ +import {fromHexString, toHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAllForks, computeEpochAtSlot, @@ -13,7 +14,6 @@ import { BLS_WITHDRAWAL_PREFIX, } from "@lodestar/params"; import {Epoch, phase0, capella, ssz, ValidatorIndex} from "@lodestar/types"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import {IBeaconDb} from "../../db/index.js"; import {SignedBLSToExecutionChangeVersioned} from "../../util/types.js"; import {isValidBlsToExecutionChangeForBlockInclusion} from "./utils.js"; @@ -169,7 +169,7 @@ export class OpPool { phase0.AttesterSlashing[], phase0.ProposerSlashing[], phase0.SignedVoluntaryExit[], - capella.SignedBLSToExecutionChange[] + capella.SignedBLSToExecutionChange[], ] { const {config} = state; const stateEpoch = computeEpochAtSlot(state.slot); diff --git a/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts b/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts index 69c16901dc61..03552992a72a 100644 --- a/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts +++ b/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts @@ -1,8 +1,8 @@ import {PointFormat, Signature} from "@chainsafe/bls/types"; import bls from "@chainsafe/bls"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {altair, Root, Slot, SubcommitteeIndex} from "@lodestar/types"; -import {BitArray, toHexString} from "@chainsafe/ssz"; import {MapDef} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js"; diff --git a/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts b/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts index 081bd59fb88c..51b433fd6e50 100644 --- a/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts +++ b/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts @@ -1,9 +1,9 @@ import type {Signature} from "@chainsafe/bls/types"; import bls from "@chainsafe/bls"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params"; import {altair, Slot, Root, ssz} from "@lodestar/types"; import {G2_POINT_AT_INFINITY} from "@lodestar/state-transition"; -import {BitArray, toHexString} from "@chainsafe/ssz"; import {MapDef} from "@lodestar/utils"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js"; import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js"; diff --git a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts index 21e81e4ae38a..bac501ed725c 100644 --- a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts +++ b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts @@ -7,9 +7,6 @@ import { import {allForks, Root} from "@lodestar/types"; import {ZERO_HASH} from "../../constants/index.js"; import {Metrics} from "../../metrics/index.js"; -import {BlockType, AssembledBlockType} from "./produceBlockBody.js"; - -export {BlockType, AssembledBlockType}; /** * Instead of running fastStateTransition(), only need to process block since diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index c92a95428ad2..dd111f14b4d1 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -1,7 +1,7 @@ +import {toHexString} from "@chainsafe/ssz"; import {phase0, Slot, allForks, RootHex, Epoch} from "@lodestar/types"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; -import {toHexString} from "@chainsafe/ssz"; import {Logger} from "@lodestar/utils"; import {routes} from "@lodestar/api"; import {CheckpointHex, CheckpointStateCache, StateContextCache, toCheckpointHex} from "../stateCache/index.js"; diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 90145974ba65..11e54289c1d2 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -1,3 +1,4 @@ +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {allForks, phase0, Slot, RootHex} from "@lodestar/types"; import { CachedBeaconStateAllForks, @@ -8,7 +9,6 @@ import { processSlots, stateTransition, } from "@lodestar/state-transition"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {sleep} from "@lodestar/utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; diff --git a/packages/beacon-node/src/chain/seenCache/seenAggregateAndProof.ts b/packages/beacon-node/src/chain/seenCache/seenAggregateAndProof.ts index 0b4a4a1d8460..e17e6c671caa 100644 --- a/packages/beacon-node/src/chain/seenCache/seenAggregateAndProof.ts +++ b/packages/beacon-node/src/chain/seenCache/seenAggregateAndProof.ts @@ -1,5 +1,5 @@ -import {Epoch, RootHex} from "@lodestar/types"; import {BitArray} from "@chainsafe/ssz"; +import {Epoch, RootHex} from "@lodestar/types"; import {MapDef} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; import {isSuperSetOrEqual} from "../../util/bitArray.js"; diff --git a/packages/beacon-node/src/chain/seenCache/seenCommitteeContribution.ts b/packages/beacon-node/src/chain/seenCache/seenCommitteeContribution.ts index 880cb551ab0b..fedaff8225d6 100644 --- a/packages/beacon-node/src/chain/seenCache/seenCommitteeContribution.ts +++ b/packages/beacon-node/src/chain/seenCache/seenCommitteeContribution.ts @@ -1,7 +1,7 @@ +import {toHexString} from "@chainsafe/ssz"; import {Slot, ValidatorIndex} from "@lodestar/types"; import {ContributionAndProof, SyncCommitteeContribution} from "@lodestar/types/altair"; import {MapDef} from "@lodestar/utils"; -import {toHexString} from "@chainsafe/ssz"; import {Metrics} from "../../metrics/index.js"; import {isSuperSetOrEqual} from "../../util/bitArray.js"; import {AggregationInfo, insertDesc} from "./seenAggregateAndProof.js"; diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 713f599dae78..dc3a963d3bdc 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -1,7 +1,7 @@ +import {toHexString} from "@chainsafe/ssz"; import {phase0, Epoch, Root, Slot, RootHex, ssz} from "@lodestar/types"; import {ProtoBlock} from "@lodestar/fork-choice"; import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH} from "@lodestar/params"; -import {toHexString} from "@chainsafe/ssz"; import { computeEpochAtSlot, CachedBeaconStateAllForks, diff --git a/packages/beacon-node/src/chain/validation/block.ts b/packages/beacon-node/src/chain/validation/block.ts index 8a7b4c659dc8..daf99cf5365c 100644 --- a/packages/beacon-node/src/chain/validation/block.ts +++ b/packages/beacon-node/src/chain/validation/block.ts @@ -1,3 +1,4 @@ +import {toHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {allForks} from "@lodestar/types"; import { @@ -10,7 +11,6 @@ import { } from "@lodestar/state-transition"; import {sleep} from "@lodestar/utils"; import {ForkName} from "@lodestar/params"; -import {toHexString} from "@chainsafe/ssz"; import {MAXIMUM_GOSSIP_CLOCK_DISPARITY} from "../../constants/index.js"; import {IBeaconChain} from "../interface.js"; import {BlockGossipError, BlockErrorCode, GossipAction} from "../errors/index.js"; diff --git a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts index c31b210e0f6a..099590ee019e 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts @@ -1,7 +1,7 @@ +import type {PublicKey} from "@chainsafe/bls/types"; import {DOMAIN_AGGREGATE_AND_PROOF} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {Epoch, phase0} from "@lodestar/types"; -import type {PublicKey} from "@chainsafe/bls/types"; import { CachedBeaconStateAllForks, computeSigningRoot, diff --git a/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts index 7c19091992e6..dbb8e3380606 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts @@ -1,6 +1,6 @@ +import type {PublicKey} from "@chainsafe/bls/types"; import {DOMAIN_SELECTION_PROOF} from "@lodestar/params"; import {phase0, Slot, ssz} from "@lodestar/types"; -import type {PublicKey} from "@chainsafe/bls/types"; import { CachedBeaconStateAllForks, computeSigningRoot, diff --git a/packages/beacon-node/src/chain/validation/syncCommittee.ts b/packages/beacon-node/src/chain/validation/syncCommittee.ts index 79b4bc092f66..f5450d0b26ec 100644 --- a/packages/beacon-node/src/chain/validation/syncCommittee.ts +++ b/packages/beacon-node/src/chain/validation/syncCommittee.ts @@ -1,7 +1,7 @@ +import {toHexString} from "@chainsafe/ssz"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {SYNC_COMMITTEE_SUBNET_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {altair} from "@lodestar/types"; -import {toHexString} from "@chainsafe/ssz"; import {GossipAction, SyncCommitteeError, SyncCommitteeErrorCode} from "../errors/index.js"; import {IBeaconChain} from "../interface.js"; import {getSyncCommitteeSignatureSet} from "./signatureSets/index.js"; diff --git a/packages/beacon-node/src/db/beacon.ts b/packages/beacon-node/src/db/beacon.ts index 9fca0b55bb0e..e95fbcab922e 100644 --- a/packages/beacon-node/src/db/beacon.ts +++ b/packages/beacon-node/src/db/beacon.ts @@ -60,7 +60,10 @@ export class BeaconDb implements IBeaconDb { backfilledRanges: BackfilledRanges; - constructor(config: ChainForkConfig, protected readonly db: Db) { + constructor( + config: ChainForkConfig, + protected readonly db: Db + ) { // Warning: If code is ever run in the constructor, must change this stub to not extend 'packages/beacon-node/test/utils/stub/beaconDb.ts' - this.block = new BlockRepository(config, db); this.blockArchive = new BlockArchiveRepository(config, db); diff --git a/packages/beacon-node/src/db/buckets.ts b/packages/beacon-node/src/db/buckets.ts index 128cb5772e00..0ad10ed904e1 100644 --- a/packages/beacon-node/src/db/buckets.ts +++ b/packages/beacon-node/src/db/buckets.ts @@ -41,6 +41,9 @@ export enum Bucket { // note: below buckets would not be in use till deneb hf so their number assignments // can be ignored and safely deleted later on allForks_blobsSidecar = 29, // DENEB BeaconBlockRoot -> BlobsSidecar + + // https://github.com/ChainSafe/lodestar/issues/5753 + // eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values allForks_blobsSidecarArchive = 30, // DENEB BeaconBlockSlot -> BlobsSidecar // Lightclient server diff --git a/packages/beacon-node/src/db/repositories/blobSidecars.ts b/packages/beacon-node/src/db/repositories/blobSidecars.ts index afd1754f6e82..576a03df9e61 100644 --- a/packages/beacon-node/src/db/repositories/blobSidecars.ts +++ b/packages/beacon-node/src/db/repositories/blobSidecars.ts @@ -1,7 +1,7 @@ +import {ValueOf, ContainerType} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {Db, Repository} from "@lodestar/db"; import {ssz} from "@lodestar/types"; -import {ValueOf, ContainerType} from "@chainsafe/ssz"; import {Bucket, getBucketNameByValue} from "../buckets.js"; export const blobSidecarsWrapperSsz = new ContainerType( diff --git a/packages/beacon-node/src/db/repositories/lightclientSyncCommitteeWitness.ts b/packages/beacon-node/src/db/repositories/lightclientSyncCommitteeWitness.ts index f0385f53c424..45f91f159997 100644 --- a/packages/beacon-node/src/db/repositories/lightclientSyncCommitteeWitness.ts +++ b/packages/beacon-node/src/db/repositories/lightclientSyncCommitteeWitness.ts @@ -1,7 +1,7 @@ +import {ContainerType, VectorCompositeType} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {DatabaseController, Repository} from "@lodestar/db"; import {ssz} from "@lodestar/types"; -import {ContainerType, VectorCompositeType} from "@chainsafe/ssz"; import {SyncCommitteeWitness} from "../../chain/lightClient/types.js"; import {Bucket, getBucketNameByValue} from "../buckets.js"; diff --git a/packages/beacon-node/src/eth1/eth1DepositsCache.ts b/packages/beacon-node/src/eth1/eth1DepositsCache.ts index b2c3c0c83a11..2fb187d02cf7 100644 --- a/packages/beacon-node/src/eth1/eth1DepositsCache.ts +++ b/packages/beacon-node/src/eth1/eth1DepositsCache.ts @@ -1,5 +1,5 @@ -import {phase0, ssz} from "@lodestar/types"; import {byteArrayEquals} from "@chainsafe/ssz"; +import {phase0, ssz} from "@lodestar/types"; import {FilterOptions} from "@lodestar/db"; import {ChainForkConfig} from "@lodestar/config"; diff --git a/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts b/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts index ac0d78b3917c..8602b3058180 100644 --- a/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts +++ b/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts @@ -1,7 +1,7 @@ +import {toHexString} from "@chainsafe/ssz"; import {ChainConfig} from "@lodestar/config"; import {RootHex} from "@lodestar/types"; import {Logger, pruneSetToMax} from "@lodestar/utils"; -import {toHexString} from "@chainsafe/ssz"; import {Metrics} from "../metrics/index.js"; import {ZERO_HASH_HEX} from "../constants/index.js"; import {enumToIndexMap} from "../util/enum.js"; diff --git a/packages/beacon-node/src/eth1/index.ts b/packages/beacon-node/src/eth1/index.ts index 1e23b3016334..3e44a0a6620d 100644 --- a/packages/beacon-node/src/eth1/index.ts +++ b/packages/beacon-node/src/eth1/index.ts @@ -1,6 +1,6 @@ +import {fromHexString} from "@chainsafe/ssz"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {Root} from "@lodestar/types"; -import {fromHexString} from "@chainsafe/ssz"; import {IEth1ForBlockProduction, Eth1DataAndDeposits, IEth1Provider, PowMergeBlock, TDProgress} from "./interface.js"; import {Eth1DepositDataTracker, Eth1DepositDataTrackerModules} from "./eth1DepositDataTracker.js"; import {Eth1MergeBlockTracker, Eth1MergeBlockTrackerModules} from "./eth1MergeBlockTracker.js"; diff --git a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts index 78a6dd39f714..cf8a8bf2682b 100644 --- a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts +++ b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts @@ -317,7 +317,10 @@ export class ErrorJsonRpcResponse extends Error { /** JSON RPC endpoint returned status code != 200 */ export class HttpRpcError extends Error { - constructor(readonly status: number, message: string) { + constructor( + readonly status: number, + message: string + ) { super(message); } } diff --git a/packages/beacon-node/src/eth1/provider/utils.ts b/packages/beacon-node/src/eth1/provider/utils.ts index 6d96bb2e4381..506e4e48711a 100644 --- a/packages/beacon-node/src/eth1/provider/utils.ts +++ b/packages/beacon-node/src/eth1/provider/utils.ts @@ -1,6 +1,6 @@ +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {RootHex} from "@lodestar/types"; import {bytesToBigInt, bigIntToBytes} from "@lodestar/utils"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import {ErrorParseJson} from "./jsonRpcHttpClient.js"; /** QUANTITY as defined in ethereum execution layer JSON RPC https://eth.wiki/json-rpc/API */ diff --git a/packages/beacon-node/src/eth1/utils/deposits.ts b/packages/beacon-node/src/eth1/utils/deposits.ts index 001b6335e3dd..19544917ffdc 100644 --- a/packages/beacon-node/src/eth1/utils/deposits.ts +++ b/packages/beacon-node/src/eth1/utils/deposits.ts @@ -1,8 +1,8 @@ +import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; +import {toHexString} from "@chainsafe/ssz"; import {MAX_DEPOSITS} from "@lodestar/params"; import {BeaconStateAllForks} from "@lodestar/state-transition"; import {phase0, ssz} from "@lodestar/types"; -import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; -import {toHexString} from "@chainsafe/ssz"; import {FilterOptions} from "@lodestar/db"; import {Eth1Error, Eth1ErrorCode} from "../errors.js"; import {DepositTree} from "../../db/repositories/depositDataRoot.js"; diff --git a/packages/beacon-node/src/execution/builder/http.ts b/packages/beacon-node/src/execution/builder/http.ts index 56cc4d181d0a..39ad6062edfe 100644 --- a/packages/beacon-node/src/execution/builder/http.ts +++ b/packages/beacon-node/src/execution/builder/http.ts @@ -1,7 +1,7 @@ +import {byteArrayEquals, toHexString} from "@chainsafe/ssz"; import {allForks, bellatrix, Slot, Root, BLSPubkey, ssz, deneb, Wei} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {getClient, Api as BuilderApi} from "@lodestar/api/builder"; -import {byteArrayEquals, toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ApiError} from "@lodestar/api"; diff --git a/packages/beacon-node/src/execution/builder/index.ts b/packages/beacon-node/src/execution/builder/index.ts index 3e502dd2df88..530d541f8450 100644 --- a/packages/beacon-node/src/execution/builder/index.ts +++ b/packages/beacon-node/src/execution/builder/index.ts @@ -4,7 +4,7 @@ import {IExecutionBuilder} from "./interface.js"; import {ExecutionBuilderHttp, ExecutionBuilderHttpOpts, defaultExecutionBuilderHttpOpts} from "./http.js"; -export {IExecutionBuilder, ExecutionBuilderHttp, defaultExecutionBuilderHttpOpts}; +export {ExecutionBuilderHttp, defaultExecutionBuilderHttpOpts}; export type ExecutionBuilderOpts = {mode?: "http"} & ExecutionBuilderHttpOpts; export const defaultExecutionBuilderOpts: ExecutionBuilderOpts = defaultExecutionBuilderHttpOpts; diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index 3cf9f46297fe..bea26fa95760 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -101,7 +101,10 @@ export class ExecutionEngineHttp implements IExecutionEngine { ); }; - constructor(private readonly rpc: IJsonRpcHttpClient, {metrics, signal}: ExecutionEngineModules) { + constructor( + private readonly rpc: IJsonRpcHttpClient, + {metrics, signal}: ExecutionEngineModules + ) { this.rpcFetchQueue = new JobItemQueue<[EngineRequest], EngineResponse>( this.jobQueueProcessor, {maxLength: QUEUE_MAX_LENGTH, maxConcurrency: 1, noYieldIfOneItem: true, signal}, diff --git a/packages/beacon-node/src/execution/engine/index.ts b/packages/beacon-node/src/execution/engine/index.ts index 11d04474f533..210cba5fb489 100644 --- a/packages/beacon-node/src/execution/engine/index.ts +++ b/packages/beacon-node/src/execution/engine/index.ts @@ -11,7 +11,7 @@ import { import {ExecutionEngineMockOpts, ExecutionEngineMockBackend} from "./mock.js"; import {ExecutionEngineMockJsonRpcClient, JsonRpcBackend} from "./utils.js"; -export {IExecutionEngine, ExecutionEngineHttp, ExecutionEngineDisabled, defaultExecutionEngineHttpOpts}; +export {ExecutionEngineHttp, ExecutionEngineDisabled, defaultExecutionEngineHttpOpts}; export type ExecutionEngineOpts = | ({mode?: "http"} & ExecutionEngineHttpOpts) diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 7295e8c37575..daf66da0106b 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -35,11 +35,11 @@ export type EngineApiRpcParamTypes = { */ engine_forkchoiceUpdatedV1: [ forkChoiceData: {headBlockHash: DATA; safeBlockHash: DATA; finalizedBlockHash: DATA}, - payloadAttributes?: PayloadAttributesRpc + payloadAttributes?: PayloadAttributesRpc, ]; engine_forkchoiceUpdatedV2: [ forkChoiceData: {headBlockHash: DATA; safeBlockHash: DATA; finalizedBlockHash: DATA}, - payloadAttributes?: PayloadAttributesRpc + payloadAttributes?: PayloadAttributesRpc, ]; /** * 1. payloadId: QUANTITY, 64 Bits - Identifier of the payload building process diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts index 19a8c8d0825f..c9c62a45304c 100644 --- a/packages/beacon-node/src/network/core/networkCore.ts +++ b/packages/beacon-node/src/network/core/networkCore.ts @@ -2,15 +2,15 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {multiaddr} from "@multiformats/multiaddr"; import {Connection} from "@libp2p/interface-connection"; import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; -import {routes} from "@lodestar/api"; import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/dist/src/score/peer-score.js"; +import {fromHexString} from "@chainsafe/ssz"; +import {ENR} from "@chainsafe/discv5"; +import {routes} from "@lodestar/api"; import {BeaconConfig} from "@lodestar/config"; -import {LoggerNode} from "@lodestar/logger/node"; +import type {LoggerNode} from "@lodestar/logger/node"; import {Epoch, phase0} from "@lodestar/types"; import {ForkName} from "@lodestar/params"; import {ResponseIncoming} from "@lodestar/reqresp"; -import {fromHexString} from "@chainsafe/ssz"; -import {ENR} from "@chainsafe/discv5"; import {Libp2p} from "../interface.js"; import {PeerManager} from "../peers/peerManager.js"; import {ReqRespBeaconNode} from "../reqresp/ReqRespBeaconNode.js"; diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index 38c8ed558b1a..4f28670711c4 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -3,9 +3,9 @@ import fs from "node:fs"; import path from "node:path"; import {createFromProtobuf} from "@libp2p/peer-id-factory"; import {expose} from "@chainsafe/threads/worker"; +import type {WorkerModule} from "@chainsafe/threads/dist/types/worker.js"; import {chainConfigFromJson, createBeaconConfig} from "@lodestar/config"; import {getNodeLogger} from "@lodestar/logger/node"; -import type {WorkerModule} from "@chainsafe/threads/dist/types/worker.js"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {collectNodeJSMetrics, RegistryMetricCreator} from "../../metrics/index.js"; import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js"; diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts index d2963d5978b0..8b7a09202c17 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts @@ -1,14 +1,14 @@ import worker_threads from "node:worker_threads"; import {exportToProtobuf} from "@libp2p/peer-id-factory"; import {PeerId} from "@libp2p/interface-peer-id"; -import {routes} from "@lodestar/api"; import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/dist/src/score/peer-score.js"; -import {phase0} from "@lodestar/types"; -import {ResponseIncoming, ResponseOutgoing} from "@lodestar/reqresp"; import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; import {spawn, Thread, Worker} from "@chainsafe/threads"; +import {routes} from "@lodestar/api"; +import {phase0} from "@lodestar/types"; +import {ResponseIncoming, ResponseOutgoing} from "@lodestar/reqresp"; import {BeaconConfig, chainConfigToJson} from "@lodestar/config"; -import {LoggerNode} from "@lodestar/logger/node"; +import type {LoggerNode} from "@lodestar/logger/node"; import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js"; import {wireEventsOnMainThread} from "../../util/workerEvents.js"; import {Metrics} from "../../metrics/index.js"; diff --git a/packages/beacon-node/src/network/core/types.ts b/packages/beacon-node/src/network/core/types.ts index bb66f4dab08f..a1e8f5d9c1cc 100644 --- a/packages/beacon-node/src/network/core/types.ts +++ b/packages/beacon-node/src/network/core/types.ts @@ -1,9 +1,9 @@ +import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; +import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; import {routes} from "@lodestar/api"; import {ResponseIncoming} from "@lodestar/reqresp"; -import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; import {phase0} from "@lodestar/types"; import {LoggerNodeOpts} from "@lodestar/logger/node"; -import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; import {NetworkOptions} from "../options.js"; import {CommitteeSubscription} from "../subnets/interface.js"; import {PeerAction, PeerScoreStats} from "../peers/index.js"; diff --git a/packages/beacon-node/src/network/discv5/index.ts b/packages/beacon-node/src/network/discv5/index.ts index 33a4ac4cce5d..611898973d32 100644 --- a/packages/beacon-node/src/network/discv5/index.ts +++ b/packages/beacon-node/src/network/discv5/index.ts @@ -29,7 +29,10 @@ export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter void}; private closed = false; - constructor(private readonly opts: Discv5Opts, private readonly workerApi: Discv5WorkerApi) { + constructor( + private readonly opts: Discv5Opts, + private readonly workerApi: Discv5WorkerApi + ) { super(); this.keypair = createKeypairFromPeerId(this.opts.peerId); diff --git a/packages/beacon-node/src/network/discv5/utils.ts b/packages/beacon-node/src/network/discv5/utils.ts index 00b0a7f9dac5..70bf9517ac01 100644 --- a/packages/beacon-node/src/network/discv5/utils.ts +++ b/packages/beacon-node/src/network/discv5/utils.ts @@ -26,7 +26,7 @@ export function enrRelevance(enr: ENR, config: BeaconConfig): ENRRelevance { const forkDigest = eth2.slice(0, 4); // Check if forkDigest matches any of our known forks. const forkName = config.forkDigest2ForkNameOption(forkDigest); - if (!forkName) { + if (forkName == null) { return ENRRelevance.unknown_forkDigest; } diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index 11adcde8d834..c4ba0011bd00 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -2,9 +2,9 @@ import {compress, uncompress} from "snappyjs"; import xxhashFactory from "xxhash-wasm"; import {Message} from "@libp2p/interface-pubsub"; import {digest} from "@chainsafe/as-sha256"; +import {RPC} from "@chainsafe/libp2p-gossipsub/message"; import {intToBytes, toHex} from "@lodestar/utils"; import {ForkName} from "@lodestar/params"; -import {RPC} from "@chainsafe/libp2p-gossipsub/message"; import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js"; import {getGossipSSZType, GossipTopicCache} from "./topic.js"; @@ -62,7 +62,10 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8 } export class DataTransformSnappy { - constructor(private readonly gossipTopicCache: GossipTopicCache, private readonly maxSizePerMessage: number) {} + constructor( + private readonly gossipTopicCache: GossipTopicCache, + private readonly maxSizePerMessage: number + ) {} /** * Takes the data published by peers on a topic and transforms the data. diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index f5f807796356..5e91499fd10b 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -1,12 +1,12 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; +import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; import {BeaconConfig} from "@lodestar/config"; import {sleep} from "@lodestar/utils"; import {LoggerNode} from "@lodestar/logger/node"; import {computeStartSlotAtEpoch, computeTimeAtSlot} from "@lodestar/state-transition"; import {phase0, allForks, deneb, altair, Root, capella, SlotRootHex} from "@lodestar/types"; import {routes} from "@lodestar/api"; -import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; import {ResponseIncoming} from "@lodestar/reqresp"; import {ForkName, ForkSeq, MAX_BLOBS_PER_BLOCK} from "@lodestar/params"; import {Metrics, RegistryMetricCreator} from "../metrics/index.js"; diff --git a/packages/beacon-node/src/network/peers/discover.ts b/packages/beacon-node/src/network/peers/discover.ts index 0c64dbdd865e..fe9aefb5a949 100644 --- a/packages/beacon-node/src/network/peers/discover.ts +++ b/packages/beacon-node/src/network/peers/discover.ts @@ -1,9 +1,9 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {Multiaddr} from "@multiformats/multiaddr"; -import {PeerInfo} from "@libp2p/interface-peer-info"; +import type {PeerInfo} from "@libp2p/interface-peer-info"; +import {ENR} from "@chainsafe/discv5"; import {BeaconConfig} from "@lodestar/config"; import {pruneSetToMax, sleep} from "@lodestar/utils"; -import {ENR} from "@chainsafe/discv5"; import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {LoggerNode} from "@lodestar/logger/node"; import {NetworkCoreMetrics} from "../core/metrics.js"; diff --git a/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts b/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts index 6a255e69cbce..f7b3c24f338a 100644 --- a/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts +++ b/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts @@ -1,5 +1,5 @@ -import {ForkDigest, Root, Slot, phase0, ssz} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {ForkDigest, Root, Slot, phase0, ssz} from "@lodestar/types"; // TODO: Why this value? (From Lighthouse) const FUTURE_SLOT_TOLERANCE = 1; diff --git a/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts b/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts index e3bc724b9ff4..afe43976b63e 100644 --- a/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts +++ b/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts @@ -1,7 +1,7 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {Direction} from "@libp2p/interface-connection"; -import {altair, phase0} from "@lodestar/types"; import {BitArray} from "@chainsafe/ssz"; +import {altair, phase0} from "@lodestar/types"; import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {MapDef} from "@lodestar/utils"; import {shuffle} from "../../../util/shuffle.js"; diff --git a/packages/beacon-node/src/network/processor/index.ts b/packages/beacon-node/src/network/processor/index.ts index dbd7c56cf9df..2c02fde66a36 100644 --- a/packages/beacon-node/src/network/processor/index.ts +++ b/packages/beacon-node/src/network/processor/index.ts @@ -152,7 +152,10 @@ export class NetworkProcessor { private isProcessingCurrentSlotBlock = false; private unknownRootsBySlot = new MapDef>(() => new Set()); - constructor(modules: NetworkProcessorModules, private readonly opts: NetworkProcessorOpts) { + constructor( + modules: NetworkProcessorModules, + private readonly opts: NetworkProcessorOpts + ) { const {chain, events, logger, metrics} = modules; this.chain = chain; this.events = events; diff --git a/packages/beacon-node/src/network/reqresp/utils/collect.ts b/packages/beacon-node/src/network/reqresp/utils/collect.ts index 8786dfb6883f..06f3cfc36806 100644 --- a/packages/beacon-node/src/network/reqresp/utils/collect.ts +++ b/packages/beacon-node/src/network/reqresp/utils/collect.ts @@ -1,5 +1,5 @@ -import {ResponseIncoming, RequestErrorCode, RequestError} from "@lodestar/reqresp"; import {Type} from "@chainsafe/ssz"; +import {ResponseIncoming, RequestErrorCode, RequestError} from "@lodestar/reqresp"; import {ResponseTypeGetter} from "../types.js"; import {WithBytes} from "../../interface.js"; diff --git a/packages/beacon-node/src/node/nodejs.ts b/packages/beacon-node/src/node/nodejs.ts index 43b82a8b769d..4c8ecd4a7f96 100644 --- a/packages/beacon-node/src/node/nodejs.ts +++ b/packages/beacon-node/src/node/nodejs.ts @@ -5,7 +5,7 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {BeaconConfig} from "@lodestar/config"; import {phase0} from "@lodestar/types"; import {sleep} from "@lodestar/utils"; -import {LoggerNode} from "@lodestar/logger/node"; +import type {LoggerNode} from "@lodestar/logger/node"; import {Api, ServerApi} from "@lodestar/api"; import {BeaconStateAllForks} from "@lodestar/state-transition"; import {ProcessShutdownCallback} from "@lodestar/validator"; diff --git a/packages/beacon-node/src/node/utils/interop/deposits.ts b/packages/beacon-node/src/node/utils/interop/deposits.ts index 191a045940f1..6cce7e883b84 100644 --- a/packages/beacon-node/src/node/utils/interop/deposits.ts +++ b/packages/beacon-node/src/node/utils/interop/deposits.ts @@ -1,6 +1,6 @@ import {digest} from "@chainsafe/as-sha256"; -import {phase0, ssz} from "@lodestar/types"; import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; +import {phase0, ssz} from "@lodestar/types"; import {ChainConfig} from "@lodestar/config"; import {computeDomain, computeSigningRoot, interopSecretKeys, ZERO_HASH} from "@lodestar/state-transition"; import { diff --git a/packages/beacon-node/src/sync/backfill/backfill.ts b/packages/beacon-node/src/sync/backfill/backfill.ts index a93edce53d0b..7d06f1dc7b65 100644 --- a/packages/beacon-node/src/sync/backfill/backfill.ts +++ b/packages/beacon-node/src/sync/backfill/backfill.ts @@ -1,10 +1,10 @@ import {EventEmitter} from "events"; import {StrictEventEmitter} from "strict-event-emitter-types"; +import {toHexString} from "@chainsafe/ssz"; import {BeaconStateAllForks, blockToHeader} from "@lodestar/state-transition"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {phase0, Root, Slot, allForks, ssz} from "@lodestar/types"; import {ErrorAborted, Logger, sleep, toHex} from "@lodestar/utils"; -import {toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {IBeaconChain} from "../../chain/index.js"; @@ -851,7 +851,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} backfilled: this.syncAnchor.lastBackSyncedBlock.slot, }); } - if (error) throw new BackfillSyncError({code: error}); + if (error != null) throw new BackfillSyncError({code: error}); } } diff --git a/packages/beacon-node/src/sync/range/chain.ts b/packages/beacon-node/src/sync/range/chain.ts index 6652951c876c..4fcf91d1b3e9 100644 --- a/packages/beacon-node/src/sync/range/chain.ts +++ b/packages/beacon-node/src/sync/range/chain.ts @@ -1,7 +1,7 @@ +import {toHexString} from "@chainsafe/ssz"; import {Epoch, Root, Slot, phase0} from "@lodestar/types"; import {ErrorAborted, Logger} from "@lodestar/utils"; import {ChainForkConfig} from "@lodestar/config"; -import {toHexString} from "@chainsafe/ssz"; import {BlockInput} from "../../chain/blocks/types.js"; import {PeerAction} from "../../network/index.js"; import {ItTrigger} from "../../util/itTrigger.js"; diff --git a/packages/beacon-node/src/sync/range/utils/chainTarget.ts b/packages/beacon-node/src/sync/range/utils/chainTarget.ts index e7c4c0bb3164..221b6e1be68f 100644 --- a/packages/beacon-node/src/sync/range/utils/chainTarget.ts +++ b/packages/beacon-node/src/sync/range/utils/chainTarget.ts @@ -1,5 +1,5 @@ -import {Root, Slot} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {Root, Slot} from "@lodestar/types"; /** * Sync this up to this target. Uses slot instead of epoch to re-use logic for finalized sync diff --git a/packages/beacon-node/src/sync/unknownBlock.ts b/packages/beacon-node/src/sync/unknownBlock.ts index 32d8070c4fa3..b41d0c4cb06e 100644 --- a/packages/beacon-node/src/sync/unknownBlock.ts +++ b/packages/beacon-node/src/sync/unknownBlock.ts @@ -1,7 +1,7 @@ +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {Logger, pruneSetToMax} from "@lodestar/utils"; import {Root, RootHex} from "@lodestar/types"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import {INTERVALS_PER_SLOT} from "@lodestar/params"; import {sleep} from "@lodestar/utils"; import {INetwork, NetworkEvent, NetworkEventData, PeerAction} from "../network/index.js"; diff --git a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts index e4e767107ecb..716c4d196367 100644 --- a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts @@ -1,4 +1,5 @@ import {expect} from "chai"; +import bls from "@chainsafe/bls"; import {createBeaconConfig, ChainConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; import {ApiError, getClient, routes} from "@lodestar/api"; @@ -6,7 +7,6 @@ import {sleep} from "@lodestar/utils"; import {ForkName, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; import {Validator} from "@lodestar/validator"; import {phase0, ssz} from "@lodestar/types"; -import bls from "@chainsafe/bls"; import {LogLevel, testLogger, TestLoggerOpts} from "../../../../utils/logger.js"; import {getDevBeaconNode} from "../../../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/chain/lightclient.test.ts b/packages/beacon-node/test/e2e/chain/lightclient.test.ts index c176784719b6..d5b645b39dd0 100644 --- a/packages/beacon-node/test/e2e/chain/lightclient.test.ts +++ b/packages/beacon-node/test/e2e/chain/lightclient.test.ts @@ -1,8 +1,8 @@ import {expect} from "chai"; -import {ChainConfig} from "@lodestar/config"; -import {ssz, altair} from "@lodestar/types"; import {JsonPath, toHexString, fromHexString} from "@chainsafe/ssz"; import {computeDescriptor, TreeOffsetProof} from "@chainsafe/persistent-merkle-tree"; +import {ChainConfig} from "@lodestar/config"; +import {ssz, altair} from "@lodestar/types"; import {TimestampFormatCode} from "@lodestar/logger"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params"; import {Lightclient} from "@lodestar/light-client"; diff --git a/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts b/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts index 0dec0192eed2..610072ebafe3 100644 --- a/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts +++ b/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts @@ -1,9 +1,9 @@ import {expect} from "chai"; +import {fromHexString} from "@chainsafe/ssz"; import {routes} from "@lodestar/api/beacon"; import {BLSPubkey, Epoch, phase0, Slot, ssz} from "@lodestar/types"; import {ChainConfig} from "@lodestar/config"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; -import {fromHexString} from "@chainsafe/ssz"; import {Validator} from "@lodestar/validator"; import {PubkeyHex} from "@lodestar/validator/src/types"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/eth1/eth1ForBlockProduction.test.ts b/packages/beacon-node/test/e2e/eth1/eth1ForBlockProduction.test.ts index 85450052dc56..cb537b14563f 100644 --- a/packages/beacon-node/test/e2e/eth1/eth1ForBlockProduction.test.ts +++ b/packages/beacon-node/test/e2e/eth1/eth1ForBlockProduction.test.ts @@ -2,10 +2,10 @@ import "mocha"; import {promisify} from "node:util"; import {expect} from "chai"; import leveldown from "leveldown"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {sleep} from "@lodestar/utils"; import {LevelDbController} from "@lodestar/db"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import {ssz} from "@lodestar/types"; import {Eth1ForBlockProduction} from "../../../src/eth1/index.js"; import {Eth1Options} from "../../../src/eth1/options.js"; diff --git a/packages/beacon-node/test/e2e/eth1/eth1MergeBlockTracker.test.ts b/packages/beacon-node/test/e2e/eth1/eth1MergeBlockTracker.test.ts index b0e4819a4f3a..67d392500cd5 100644 --- a/packages/beacon-node/test/e2e/eth1/eth1MergeBlockTracker.test.ts +++ b/packages/beacon-node/test/e2e/eth1/eth1MergeBlockTracker.test.ts @@ -1,7 +1,7 @@ import {expect} from "chai"; +import {fromHexString} from "@chainsafe/ssz"; import {ChainConfig} from "@lodestar/config"; import {sleep} from "@lodestar/utils"; -import {fromHexString} from "@chainsafe/ssz"; import {Eth1Provider, IEth1Provider} from "../../../src/index.js"; import {Eth1MergeBlockTracker, StatusCode} from "../../../src/eth1/eth1MergeBlockTracker.js"; import {Eth1Options} from "../../../src/eth1/options.js"; diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts index aed350a4c76d..c714183be502 100644 --- a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts +++ b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts @@ -2,8 +2,8 @@ import {Connection} from "@libp2p/interface-connection"; import {CustomEvent} from "@libp2p/interfaces/events"; import sinon from "sinon"; import {expect} from "chai"; -import {config} from "@lodestar/config/default"; import {BitArray} from "@chainsafe/ssz"; +import {config} from "@lodestar/config/default"; import {altair, phase0, ssz} from "@lodestar/types"; import {sleep} from "@lodestar/utils"; import {createBeaconConfig} from "@lodestar/config"; diff --git a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts index 4a4a5e4e5d5b..97d4b45c7558 100644 --- a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts @@ -1,7 +1,7 @@ +import {fromHexString} from "@chainsafe/ssz"; import {ChainConfig} from "@lodestar/config"; import {phase0} from "@lodestar/types"; import {config} from "@lodestar/config/default"; -import {fromHexString} from "@chainsafe/ssz"; import {TimestampFormatCode} from "@lodestar/logger"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {routes} from "@lodestar/api"; diff --git a/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts b/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts index 68784032f5f0..10527fdf94b5 100644 --- a/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts +++ b/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts @@ -1,5 +1,6 @@ import {itBench} from "@dapplion/benchmark"; import {PointFormat} from "@chainsafe/bls/types"; +// eslint-disable-next-line import/no-relative-packages import {generatePerfTestCachedStatePhase0, numValidators} from "../../../../../../state-transition/test/perf/util.js"; import {getPubkeysForIndices} from "../../../../../src/api/impl/validator/utils.js"; import {linspace} from "../../../../../src/util/numpy.js"; diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 77c7bcd9948c..309a5c29a6b0 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -1,6 +1,7 @@ import sinon from "sinon"; import {itBench} from "@dapplion/benchmark"; import {expect} from "chai"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAltair, computeEpochAtSlot, @@ -8,11 +9,11 @@ import { getBlockRootAtSlot, } from "@lodestar/state-transition"; import {HISTORICAL_ROOTS_LIMIT, SLOTS_PER_EPOCH, TIMELY_SOURCE_FLAG_INDEX} from "@lodestar/params"; -import {BitArray, toHexString} from "@chainsafe/ssz"; import {ExecutionStatus, ForkChoice, IForkChoiceStore, ProtoArray} from "@lodestar/fork-choice"; import {ssz} from "@lodestar/types"; -import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool.js"; +// eslint-disable-next-line import/no-relative-packages import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; +import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool.js"; import {computeAnchorCheckpoint} from "../../../../src/chain/initState.js"; /** Same to https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/specs/altair/beacon-chain.md#has_flag */ diff --git a/packages/beacon-node/test/perf/chain/seenCache/seenAggregateAndProof.test.ts b/packages/beacon-node/test/perf/chain/seenCache/seenAggregateAndProof.test.ts index 54eee0e907db..8ba77ca8692f 100644 --- a/packages/beacon-node/test/perf/chain/seenCache/seenAggregateAndProof.test.ts +++ b/packages/beacon-node/test/perf/chain/seenCache/seenAggregateAndProof.test.ts @@ -1,6 +1,6 @@ import {itBench} from "@dapplion/benchmark"; -import {TARGET_AGGREGATORS_PER_COMMITTEE} from "@lodestar/params"; import {BitArray} from "@chainsafe/ssz"; +import {TARGET_AGGREGATORS_PER_COMMITTEE} from "@lodestar/params"; import {SeenAggregatedAttestations} from "../../../../src/chain/seenCache/seenAggregateAndProof.js"; describe("SeenAggregatedAttestations perf test", function () { diff --git a/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts index 114a0b2ee011..6b0e9acc6c68 100644 --- a/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts @@ -1,6 +1,7 @@ import {itBench} from "@dapplion/benchmark"; -import {validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; +// eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; +import {validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; import {getAggregateAndProofValidData} from "../../../utils/validationData/aggregateAndProof.js"; describe("validate gossip signedAggregateAndProof", () => { diff --git a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts index 146b9705a5ce..ed289444f8d0 100644 --- a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts @@ -1,6 +1,7 @@ import {itBench} from "@dapplion/benchmark"; -import {validateGossipAttestation} from "../../../../src/chain/validation/index.js"; +// eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; +import {validateGossipAttestation} from "../../../../src/chain/validation/index.js"; import {getAttestationValidData} from "../../../utils/validationData/attestation.js"; describe("validate gossip attestation", () => { diff --git a/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts b/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts index 862d6d594b15..8db7deb5d3c3 100644 --- a/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts +++ b/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts @@ -4,12 +4,14 @@ import {SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY, SLOTS_PER_EPOCH} from "@lodestar/pa import {LevelDbController} from "@lodestar/db"; import {sleep} from "@lodestar/utils"; import {defaultOptions as defaultValidatorOptions} from "@lodestar/validator"; +// eslint-disable-next-line import/no-relative-packages +import {rangeSyncTest} from "../../../../state-transition/test/perf/params.js"; import { beforeValue, getNetworkCachedState, getNetworkCachedBlock, + // eslint-disable-next-line import/no-relative-packages } from "../../../../state-transition/test/utils/index.js"; -import {rangeSyncTest} from "../../../../state-transition/test/perf/params.js"; import {BeaconChain} from "../../../src/chain/index.js"; import {ExecutionEngineDisabled} from "../../../src/execution/engine/index.js"; import {Eth1ForBlockProductionDisabled} from "../../../src/eth1/index.js"; diff --git a/packages/beacon-node/test/perf/eth1/pickEth1Vote.test.ts b/packages/beacon-node/test/perf/eth1/pickEth1Vote.test.ts index 80ff464edd71..20efeefe0923 100644 --- a/packages/beacon-node/test/perf/eth1/pickEth1Vote.test.ts +++ b/packages/beacon-node/test/perf/eth1/pickEth1Vote.test.ts @@ -1,6 +1,6 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; -import {phase0, ssz} from "@lodestar/types"; import {ContainerType, ListCompositeType} from "@chainsafe/ssz"; +import {phase0, ssz} from "@lodestar/types"; import {newFilledArray, BeaconStateAllForks} from "@lodestar/state-transition"; import {fastSerializeEth1Data, pickEth1Vote} from "../../../src/eth1/utils/eth1Vote.js"; diff --git a/packages/beacon-node/test/perf/misc/throw.test.ts b/packages/beacon-node/test/perf/misc/throw.test.ts index 5d8b77a6cd7a..20f88a0947f8 100644 --- a/packages/beacon-node/test/perf/misc/throw.test.ts +++ b/packages/beacon-node/test/perf/misc/throw.test.ts @@ -13,7 +13,10 @@ describe("misc / throw vs return", () => { } class ErrorStatus extends Error implements Status { - constructor(readonly code: string, readonly value: number) { + constructor( + readonly code: string, + readonly value: number + ) { super(code); } } diff --git a/packages/beacon-node/test/scripts/blsPubkeyBytesFrequency.ts b/packages/beacon-node/test/scripts/blsPubkeyBytesFrequency.ts index d98197a33a7a..c8cd742a6d3f 100644 --- a/packages/beacon-node/test/scripts/blsPubkeyBytesFrequency.ts +++ b/packages/beacon-node/test/scripts/blsPubkeyBytesFrequency.ts @@ -1,8 +1,8 @@ import fs from "node:fs"; +import {digest} from "@chainsafe/as-sha256"; import {ApiError, getClient} from "@lodestar/api"; import {config} from "@lodestar/config/default"; import {newZeroedArray} from "@lodestar/state-transition"; -import {digest} from "@chainsafe/as-sha256"; // Script to analyze if a raw BLS pubkey bytes are sufficiently even distributed. // If so, a shorter slice of the pubkey bytes can be used as key for the pubkey to index map. diff --git a/packages/beacon-node/test/spec/bls/bls.ts b/packages/beacon-node/test/spec/bls/bls.ts index 651ab5c59985..c8d42ad84c5f 100644 --- a/packages/beacon-node/test/spec/bls/bls.ts +++ b/packages/beacon-node/test/spec/bls/bls.ts @@ -1,7 +1,7 @@ import bls from "@chainsafe/bls"; import {CoordType} from "@chainsafe/bls/types"; -import {toHexString} from "@lodestar/utils"; import {fromHexString} from "@chainsafe/ssz"; +import {toHexString} from "@lodestar/utils"; /* eslint-disable @typescript-eslint/naming-convention */ diff --git a/packages/beacon-node/test/spec/general/bls.ts b/packages/beacon-node/test/spec/general/bls.ts index 3b5ea2cde315..88c1d79bcb79 100644 --- a/packages/beacon-node/test/spec/general/bls.ts +++ b/packages/beacon-node/test/spec/general/bls.ts @@ -1,8 +1,8 @@ import bls from "@chainsafe/bls"; import {CoordType} from "@chainsafe/bls/types"; +import {fromHexString} from "@chainsafe/ssz"; import {InputType} from "@lodestar/spec-test-util"; import {toHexString} from "@lodestar/utils"; -import {fromHexString} from "@chainsafe/ssz"; import {TestRunnerFn} from "../utils/types.js"; /* eslint-disable @typescript-eslint/naming-convention */ diff --git a/packages/beacon-node/test/spec/presets/fork_choice.ts b/packages/beacon-node/test/spec/presets/fork_choice.ts index 5fae3acc3a5e..dae951906811 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.ts @@ -1,7 +1,7 @@ import {expect} from "chai"; +import {toHexString} from "@chainsafe/ssz"; import {BeaconStateAllForks, isExecutionStateType} from "@lodestar/state-transition"; import {InputType} from "@lodestar/spec-test-util"; -import {toHexString} from "@chainsafe/ssz"; import {CheckpointWithHex, ForkChoice} from "@lodestar/fork-choice"; import {phase0, allForks, bellatrix, ssz, RootHex, deneb} from "@lodestar/types"; import {bnToNum} from "@lodestar/utils"; diff --git a/packages/beacon-node/test/spec/presets/light_client/single_merkle_proof.ts b/packages/beacon-node/test/spec/presets/light_client/single_merkle_proof.ts index c347a7831001..b8df2f01a8f0 100644 --- a/packages/beacon-node/test/spec/presets/light_client/single_merkle_proof.ts +++ b/packages/beacon-node/test/spec/presets/light_client/single_merkle_proof.ts @@ -1,9 +1,9 @@ import {expect} from "chai"; +import {Tree} from "@chainsafe/persistent-merkle-tree"; +import {TreeViewDU, Type} from "@chainsafe/ssz"; import {RootHex, ssz} from "@lodestar/types"; import {InputType} from "@lodestar/spec-test-util"; import {ForkName} from "@lodestar/params"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; -import {TreeViewDU, Type} from "@chainsafe/ssz"; import {toHex} from "@lodestar/utils"; import {TestRunnerFn} from "../../utils/types.js"; diff --git a/packages/beacon-node/test/spec/presets/light_client/sync.ts b/packages/beacon-node/test/spec/presets/light_client/sync.ts index 6c2cbd27379b..8f4d5dc59056 100644 --- a/packages/beacon-node/test/spec/presets/light_client/sync.ts +++ b/packages/beacon-node/test/spec/presets/light_client/sync.ts @@ -1,7 +1,7 @@ import {expect} from "chai"; +import {init} from "@chainsafe/bls/switchable"; import {isForkLightClient} from "@lodestar/params"; import {altair, phase0, RootHex, Slot, ssz} from "@lodestar/types"; -import {init} from "@chainsafe/bls/switchable"; import {InputType} from "@lodestar/spec-test-util"; import {createBeaconConfig, ChainConfig} from "@lodestar/config"; import {fromHex, toHex} from "@lodestar/utils"; diff --git a/packages/beacon-node/test/spec/presets/merkle.ts b/packages/beacon-node/test/spec/presets/merkle.ts index 7a74d73a0a94..8570e1ca4663 100644 --- a/packages/beacon-node/test/spec/presets/merkle.ts +++ b/packages/beacon-node/test/spec/presets/merkle.ts @@ -1,9 +1,9 @@ import {expect} from "chai"; +import {ProofType, SingleProof, Tree} from "@chainsafe/persistent-merkle-tree"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {ssz} from "@lodestar/types"; import {BeaconStateAllForks} from "@lodestar/state-transition"; import {InputType} from "@lodestar/spec-test-util"; -import {ProofType, SingleProof, Tree} from "@chainsafe/persistent-merkle-tree"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import {verifyMerkleBranch} from "@lodestar/utils"; import {TestRunnerFn} from "../utils/types.js"; diff --git a/packages/beacon-node/test/spec/presets/rewards.ts b/packages/beacon-node/test/spec/presets/rewards.ts index 28a5ef7de387..9352b10a7737 100644 --- a/packages/beacon-node/test/spec/presets/rewards.ts +++ b/packages/beacon-node/test/spec/presets/rewards.ts @@ -1,7 +1,7 @@ import {expect} from "chai"; +import {VectorCompositeType} from "@chainsafe/ssz"; import {BeaconStateAllForks, beforeProcessEpoch} from "@lodestar/state-transition"; import {getRewardsAndPenalties} from "@lodestar/state-transition/epoch"; -import {VectorCompositeType} from "@chainsafe/ssz"; import {ssz} from "@lodestar/types"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; import {inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; diff --git a/packages/beacon-node/test/spec/presets/ssz_static.ts b/packages/beacon-node/test/spec/presets/ssz_static.ts index 8bcf8f01959b..014e7d6293e0 100644 --- a/packages/beacon-node/test/spec/presets/ssz_static.ts +++ b/packages/beacon-node/test/spec/presets/ssz_static.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; -import {ssz} from "@lodestar/types"; import {Type} from "@chainsafe/ssz"; +import {ssz} from "@lodestar/types"; import {ACTIVE_PRESET, ForkName, ForkLightClient} from "@lodestar/params"; import {replaceUintTypeWithUintBigintType} from "../utils/replaceUintTypeWithUintBigintType.js"; import {parseSszStaticTestcase} from "../utils/sszTestCaseParser.js"; diff --git a/packages/beacon-node/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts b/packages/beacon-node/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts index f3451a0a81e1..45422eb3f7e6 100644 --- a/packages/beacon-node/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts +++ b/packages/beacon-node/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts @@ -1,6 +1,6 @@ import {expect} from "chai"; -import {ssz} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {ssz} from "@lodestar/types"; import {generateProtoBlock, generateSignedBlockAtSlot} from "../../../../../utils/typeGenerator.js"; import {setupApiImplTestServer, ApiImplTestModules} from "../../index.test.js"; diff --git a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts index e765fa323dca..94972be77c4f 100644 --- a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts +++ b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts @@ -1,7 +1,7 @@ import {expect, use} from "chai"; import chaiAsPromised from "chai-as-promised"; -import {phase0} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {phase0} from "@lodestar/types"; import {getValidatorStatus, getStateValidatorIndex} from "../../../../../../src/api/impl/beacon/state/utils.js"; import {generateCachedAltairState} from "../../../../../utils/state.js"; diff --git a/packages/beacon-node/test/unit/chain/archive/blockArchiver.test.ts b/packages/beacon-node/test/unit/chain/archive/blockArchiver.test.ts index 469e53d8cfcf..4ddca3ed90f6 100644 --- a/packages/beacon-node/test/unit/chain/archive/blockArchiver.test.ts +++ b/packages/beacon-node/test/unit/chain/archive/blockArchiver.test.ts @@ -1,9 +1,9 @@ import {expect} from "chai"; import sinon, {SinonStubbedInstance} from "sinon"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {ssz} from "@lodestar/types"; import {ForkChoice} from "@lodestar/fork-choice"; import {config} from "@lodestar/config/default"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import {ZERO_HASH_HEX} from "../../../../src/constants/index.js"; import {generateProtoBlock} from "../../../utils/typeGenerator.js"; import {StubbedBeaconDb} from "../../../utils/stub/index.js"; diff --git a/packages/beacon-node/test/unit/chain/genesis/genesis.test.ts b/packages/beacon-node/test/unit/chain/genesis/genesis.test.ts index 23e29abaa090..78be9a88e4be 100644 --- a/packages/beacon-node/test/unit/chain/genesis/genesis.test.ts +++ b/packages/beacon-node/test/unit/chain/genesis/genesis.test.ts @@ -1,12 +1,12 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {expect} from "chai"; import type {SecretKey, PublicKey} from "@chainsafe/bls/types"; +import {toHexString} from "@chainsafe/ssz"; import {DOMAIN_DEPOSIT, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; import {config} from "@lodestar/config/default"; import {computeDomain, computeSigningRoot, interopSecretKey, ZERO_HASH} from "@lodestar/state-transition"; import {ValidatorIndex, phase0, ssz} from "@lodestar/types"; import {ErrorAborted} from "@lodestar/utils"; -import {toHexString} from "@chainsafe/ssz"; import {GenesisBuilder} from "../../../../src/chain/genesis/genesis.js"; import {testLogger} from "../../../utils/logger.js"; import {ZERO_HASH_HEX} from "../../../../src/constants/index.js"; diff --git a/packages/beacon-node/test/unit/chain/opPools/syncCommittee.test.ts b/packages/beacon-node/test/unit/chain/opPools/syncCommittee.test.ts index ef168579475a..8f6eb39241ed 100644 --- a/packages/beacon-node/test/unit/chain/opPools/syncCommittee.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/syncCommittee.test.ts @@ -1,8 +1,8 @@ import {expect} from "chai"; import sinon, {SinonStubbedInstance} from "sinon"; import bls from "@chainsafe/bls"; -import {altair} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {altair} from "@lodestar/types"; import {SyncCommitteeMessagePool} from "../../../../src/chain/opPools/index.js"; import {Clock} from "../../../../src/util/clock.js"; diff --git a/packages/beacon-node/test/unit/chain/opPools/syncCommitteeContribution.test.ts b/packages/beacon-node/test/unit/chain/opPools/syncCommitteeContribution.test.ts index 799a2a68676a..dce58ea73f9a 100644 --- a/packages/beacon-node/test/unit/chain/opPools/syncCommitteeContribution.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/syncCommitteeContribution.test.ts @@ -1,9 +1,9 @@ import {expect} from "chai"; -import {ssz} from "@lodestar/types"; -import {newFilledArray} from "@lodestar/state-transition"; import type {SecretKey} from "@chainsafe/bls/types"; import bls from "@chainsafe/bls"; import {BitArray} from "@chainsafe/ssz"; +import {newFilledArray} from "@lodestar/state-transition"; +import {ssz} from "@lodestar/types"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import { aggregate, diff --git a/packages/beacon-node/test/unit/chain/stateCache/stateContextCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/stateContextCache.test.ts index a426e53b138c..2ad38f8e93cb 100644 --- a/packages/beacon-node/test/unit/chain/stateCache/stateContextCache.test.ts +++ b/packages/beacon-node/test/unit/chain/stateCache/stateContextCache.test.ts @@ -1,6 +1,6 @@ import {expect} from "chai"; -import {EpochShuffling} from "@lodestar/state-transition"; import {toHexString} from "@chainsafe/ssz"; +import {EpochShuffling} from "@lodestar/state-transition"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {Root} from "@lodestar/types"; import {StateContextCache} from "../../../../src/chain/stateCache/index.js"; diff --git a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts index 994e11a34d3c..f3b934ac2242 100644 --- a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts @@ -2,11 +2,12 @@ import {toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {phase0, ssz} from "@lodestar/types"; import {processSlots} from "@lodestar/state-transition"; +// eslint-disable-next-line import/no-relative-packages +import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {IBeaconChain} from "../../../../src/chain/index.js"; import {AttestationErrorCode} from "../../../../src/chain/errors/index.js"; import {validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; import {expectRejectedWithLodestarError} from "../../../utils/errors.js"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {memoOnce} from "../../../utils/cache.js"; import { getAggregateAndProofValidData, diff --git a/packages/beacon-node/test/unit/chain/validation/attestation.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation.test.ts index 007882d5a609..d56cb261a8a5 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation.test.ts @@ -1,11 +1,13 @@ import sinon, {SinonStubbedInstance} from "sinon"; import {expect} from "chai"; -import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {BitArray} from "@chainsafe/ssz"; +import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch, processSlots} from "@lodestar/state-transition"; import {defaultChainConfig, createChainForkConfig, BeaconConfig} from "@lodestar/config"; import {Slot, ssz} from "@lodestar/types"; import {ProtoBlock} from "@lodestar/fork-choice"; +// eslint-disable-next-line import/no-relative-packages +import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {IBeaconChain} from "../../../../src/chain/index.js"; import {AttestationErrorCode, GossipErrorCode} from "../../../../src/chain/errors/index.js"; import { @@ -14,7 +16,6 @@ import { validateGossipAttestation, } from "../../../../src/chain/validation/index.js"; import {expectRejectedWithLodestarError} from "../../../utils/errors.js"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {memoOnce} from "../../../utils/cache.js"; import {getAttestationValidData, AttestationValidDataOpts} from "../../../utils/validationData/attestation.js"; import {IStateRegenerator, RegenCaller} from "../../../../src/chain/regen/interface.js"; diff --git a/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts b/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts index 3a1c6914c0ff..e3f3c0295655 100644 --- a/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts @@ -1,5 +1,7 @@ import sinon, {SinonStubbedInstance} from "sinon"; import {digest} from "@chainsafe/as-sha256"; +import bls from "@chainsafe/bls"; +import {PointFormat} from "@chainsafe/bls/types"; import {config as defaultConfig} from "@lodestar/config/default"; import {computeSigningRoot} from "@lodestar/state-transition"; import {ForkChoice} from "@lodestar/fork-choice"; @@ -12,8 +14,6 @@ import { SLOTS_PER_EPOCH, ForkName, } from "@lodestar/params"; -import bls from "@chainsafe/bls"; -import {PointFormat} from "@chainsafe/bls/types"; import {createBeaconConfig} from "@lodestar/config"; import {BeaconChain} from "../../../../src/chain/index.js"; diff --git a/packages/beacon-node/test/unit/chain/validation/syncCommittee.test.ts b/packages/beacon-node/test/unit/chain/validation/syncCommittee.test.ts index 1836f64d4a36..56afb8715d6d 100644 --- a/packages/beacon-node/test/unit/chain/validation/syncCommittee.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/syncCommittee.test.ts @@ -1,10 +1,10 @@ import sinon from "sinon"; import {SinonStubbedInstance} from "sinon"; import {expect} from "chai"; +import {toHexString} from "@chainsafe/ssz"; import {altair, Epoch, Slot} from "@lodestar/types"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {createChainForkConfig, defaultChainConfig} from "@lodestar/config"; -import {toHexString} from "@chainsafe/ssz"; import {ForkChoice, IForkChoice} from "@lodestar/fork-choice"; import {BeaconChain} from "../../../../src/chain/index.js"; import {Clock} from "../../../../src/util/clock.js"; diff --git a/packages/beacon-node/test/unit/chain/validation/voluntaryExit.test.ts b/packages/beacon-node/test/unit/chain/validation/voluntaryExit.test.ts index 82bc271afef9..eef6fcec9db8 100644 --- a/packages/beacon-node/test/unit/chain/validation/voluntaryExit.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/voluntaryExit.test.ts @@ -1,5 +1,7 @@ import sinon, {SinonStubbedInstance} from "sinon"; +import bls from "@chainsafe/bls"; +import {PointFormat} from "@chainsafe/bls/types"; import {config} from "@lodestar/config/default"; import { CachedBeaconStateAllForks, @@ -11,8 +13,6 @@ import {ForkChoice} from "@lodestar/fork-choice"; import {phase0, ssz} from "@lodestar/types"; import {DOMAIN_VOLUNTARY_EXIT, FAR_FUTURE_EPOCH, SLOTS_PER_EPOCH} from "@lodestar/params"; -import bls from "@chainsafe/bls"; -import {PointFormat} from "@chainsafe/bls/types"; import {createBeaconConfig} from "@lodestar/config"; import {BeaconChain} from "../../../../src/chain/index.js"; import {StubbedChainMutable} from "../../../utils/stub/index.js"; diff --git a/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts b/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts index 812aa0af6fb7..f9dafcf1fe7e 100644 --- a/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts +++ b/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts @@ -1,7 +1,7 @@ import {expect} from "chai"; +import {toHexString} from "@chainsafe/ssz"; import {ChainConfig} from "@lodestar/config"; import {sleep} from "@lodestar/utils"; -import {toHexString} from "@chainsafe/ssz"; import {IEth1Provider} from "../../../src/index.js"; import {ZERO_HASH} from "../../../src/constants/index.js"; import {Eth1MergeBlockTracker, StatusCode, toPowBlock} from "../../../src/eth1/eth1MergeBlockTracker.js"; diff --git a/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts b/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts index a6745a764c18..7b66a9248925 100644 --- a/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts +++ b/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts @@ -86,7 +86,7 @@ describe("eth1 / util / deposits", function () { if (expectedReturnedIndexes) { const result = await resultPromise; expect(result.map((deposit) => deposit.index)).to.deep.equal(expectedReturnedIndexes); - } else if (error) { + } else if (error != null) { await expectRejectedWithLodestarError(resultPromise, error); } else { throw Error("Test case must have 'result' or 'error'"); diff --git a/packages/beacon-node/test/unit/eth1/utils/eth1Data.test.ts b/packages/beacon-node/test/unit/eth1/utils/eth1Data.test.ts index 922acede85d8..05548d8b1242 100644 --- a/packages/beacon-node/test/unit/eth1/utils/eth1Data.test.ts +++ b/packages/beacon-node/test/unit/eth1/utils/eth1Data.test.ts @@ -109,7 +109,7 @@ describe("eth1 / util / getEth1DataForBlocks", function () { const eth1Datas = await eth1DatasPromise; const eth1DatasPartial = eth1Datas.map((eth1Data) => pick(eth1Data, Object.keys(expectedEth1Data[0]))); expect(eth1DatasPartial).to.deep.equal(expectedEth1Data); - } else if (error) { + } else if (error != null) { await expectRejectedWithLodestarError(eth1DatasPromise, error); } else { throw Error("Test case must have 'expectedEth1Data' or 'error'"); diff --git a/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts b/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts index ee21afb4b02b..1b4735bbea88 100644 --- a/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts @@ -1,7 +1,7 @@ import {expect} from "chai"; import {fastify} from "fastify"; -import {ForkName} from "@lodestar/params"; import {fromHexString} from "@chainsafe/ssz"; +import {ForkName} from "@lodestar/params"; import {defaultExecutionEngineHttpOpts} from "../../../src/execution/engine/http.js"; import {bytesToData, numToQuantity} from "../../../src/eth1/provider/utils.js"; diff --git a/packages/beacon-node/test/unit/network/peers/utils/enrSubnets.test.ts b/packages/beacon-node/test/unit/network/peers/utils/enrSubnets.test.ts index 5d3be91a08ef..94bb45bd0c53 100644 --- a/packages/beacon-node/test/unit/network/peers/utils/enrSubnets.test.ts +++ b/packages/beacon-node/test/unit/network/peers/utils/enrSubnets.test.ts @@ -1,8 +1,8 @@ import {expect} from "chai"; +import {BitArray} from "@chainsafe/ssz"; import {SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; -import {BitArray} from "@chainsafe/ssz"; import {deserializeEnrSubnets} from "../../../../../src/network/peers/utils/enrSubnetsDeserialize.js"; describe("ENR syncnets", () => { diff --git a/packages/beacon-node/test/unit/network/reqresp/utils.ts b/packages/beacon-node/test/unit/network/reqresp/utils.ts index e0f88955b4e2..d0e9b86832c4 100644 --- a/packages/beacon-node/test/unit/network/reqresp/utils.ts +++ b/packages/beacon-node/test/unit/network/reqresp/utils.ts @@ -1,8 +1,8 @@ import {expect} from "chai"; import {Stream, StreamStat} from "@libp2p/interface-connection"; import {Uint8ArrayList} from "uint8arraylist"; -import {Root} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {Root} from "@lodestar/types"; export function generateRoots(count: number, offset = 0): Root[] { const roots: Root[] = []; diff --git a/packages/beacon-node/test/unit/network/util.test.ts b/packages/beacon-node/test/unit/network/util.test.ts index 8b85fb8ce7da..40e05e012252 100644 --- a/packages/beacon-node/test/unit/network/util.test.ts +++ b/packages/beacon-node/test/unit/network/util.test.ts @@ -1,8 +1,8 @@ import {expect} from "chai"; import {createSecp256k1PeerId} from "@libp2p/peer-id-factory"; +import {generateKeypair, KeypairType, SignableENR} from "@chainsafe/discv5"; import {config} from "@lodestar/config/default"; import {ForkName} from "@lodestar/params"; -import {generateKeypair, KeypairType, SignableENR} from "@chainsafe/discv5"; import {defaultNetworkOptions} from "../../../src/network/options.js"; import {createNodeJsLibp2p} from "../../../src/network/index.js"; import {getCurrentAndNextFork} from "../../../src/network/forks.js"; diff --git a/packages/beacon-node/test/unit/sync/backfill/verify.test.ts b/packages/beacon-node/test/unit/sync/backfill/verify.test.ts index 18ce67fab40b..6c7d59d2b6d9 100644 --- a/packages/beacon-node/test/unit/sync/backfill/verify.test.ts +++ b/packages/beacon-node/test/unit/sync/backfill/verify.test.ts @@ -46,7 +46,7 @@ describe("backfill sync - verify block sequence", function () { blocks.filter((b) => b.data.message.slot !== 2).slice(0, blocks.length - 2), blocks[blocks.length - 1].data.message.parentRoot ); - if (error) throw new BackfillSyncError({code: error}); + if (error != null) throw new BackfillSyncError({code: error}); }).to.throw(BackfillSyncErrorCode.NOT_LINEAR); }); diff --git a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts index 0e942e5b8ce0..3bbc72e98f52 100644 --- a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts +++ b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts @@ -1,11 +1,11 @@ import {expect} from "chai"; import sinon from "sinon"; +import {toHexString} from "@chainsafe/ssz"; import {config as minimalConfig} from "@lodestar/config/default"; import {createChainForkConfig} from "@lodestar/config"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {ssz} from "@lodestar/types"; import {notNullish, sleep} from "@lodestar/utils"; -import {toHexString} from "@chainsafe/ssz"; import {IBeaconChain} from "../../../src/chain/index.js"; import {INetwork, NetworkEvent, NetworkEventBus, PeerAction} from "../../../src/network/index.js"; import {UnknownBlockSync} from "../../../src/sync/unknownBlock.js"; @@ -144,7 +144,7 @@ describe("sync / UnknownBlockSync", () => { const forkChoiceKnownRoots = new Set([blockRootHex0]); const forkChoice: Pick = { hasBlock: (root) => forkChoiceKnownRoots.has(toHexString(root)), - getFinalizedBlock: () => ({slot: finalizedSlot} as ProtoBlock), + getFinalizedBlock: () => ({slot: finalizedSlot}) as ProtoBlock, }; const seenBlockProposers: Pick = { // only return seenBlock for blockC diff --git a/packages/beacon-node/test/unit/sync/utils/remoteSyncType.test.ts b/packages/beacon-node/test/unit/sync/utils/remoteSyncType.test.ts index 2cdc351c4da5..bd9c552417cd 100644 --- a/packages/beacon-node/test/unit/sync/utils/remoteSyncType.test.ts +++ b/packages/beacon-node/test/unit/sync/utils/remoteSyncType.test.ts @@ -1,7 +1,7 @@ import {expect} from "chai"; +import {toHexString} from "@chainsafe/ssz"; import {IForkChoice} from "@lodestar/fork-choice"; import {Root, phase0} from "@lodestar/types"; -import {toHexString} from "@chainsafe/ssz"; import {ZERO_HASH} from "../../../../src/constants/index.js"; import { getPeerSyncType, diff --git a/packages/beacon-node/test/utils/clock.ts b/packages/beacon-node/test/utils/clock.ts index 19ef420649b8..ea14c866c1b0 100644 --- a/packages/beacon-node/test/utils/clock.ts +++ b/packages/beacon-node/test/utils/clock.ts @@ -4,7 +4,10 @@ import {Slot, Epoch} from "@lodestar/types"; import {IClock} from "../../src/util/clock.js"; export class ClockStatic extends EventEmitter implements IClock { - constructor(readonly currentSlot: Slot, public genesisTime = 0) { + constructor( + readonly currentSlot: Slot, + public genesisTime = 0 + ) { super(); } diff --git a/packages/beacon-node/test/utils/network.ts b/packages/beacon-node/test/utils/network.ts index 034b2b5c2d1e..a514335594db 100644 --- a/packages/beacon-node/test/utils/network.ts +++ b/packages/beacon-node/test/utils/network.ts @@ -1,7 +1,7 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {createSecp256k1PeerId} from "@libp2p/peer-id-factory"; -import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {BitArray} from "@chainsafe/ssz"; +import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; import { diff --git a/packages/beacon-node/test/utils/node/simTest.ts b/packages/beacon-node/test/utils/node/simTest.ts index 891e1a278a5a..4bf922cfb377 100644 --- a/packages/beacon-node/test/utils/node/simTest.ts +++ b/packages/beacon-node/test/utils/node/simTest.ts @@ -1,3 +1,4 @@ +import {toHexString} from "@chainsafe/ssz"; import { computeEpochAtSlot, computeStartSlotAtEpoch, @@ -10,7 +11,6 @@ import {allForks, Epoch, Slot} from "@lodestar/types"; import {Checkpoint} from "@lodestar/types/phase0"; import {Logger, mapValues} from "@lodestar/utils"; import {routes} from "@lodestar/api"; -import {toHexString} from "@chainsafe/ssz"; import {BeaconNode} from "../../../src/index.js"; import {ChainEvent, HeadEventData} from "../../../src/chain/index.js"; import {linspace} from "../../../src/util/numpy.js"; diff --git a/packages/beacon-node/test/utils/node/validator.ts b/packages/beacon-node/test/utils/node/validator.ts index 34acd9982572..240e48b8a3d1 100644 --- a/packages/beacon-node/test/utils/node/validator.ts +++ b/packages/beacon-node/test/utils/node/validator.ts @@ -1,8 +1,8 @@ import tmp from "tmp"; +import type {SecretKey} from "@chainsafe/bls/types"; import {LevelDbController} from "@lodestar/db"; import {interopSecretKey} from "@lodestar/state-transition"; import {SlashingProtection, Validator, Signer, SignerType, ValidatorProposerConfig} from "@lodestar/validator"; -import type {SecretKey} from "@chainsafe/bls/types"; import {ServerApi, Api, HttpStatusCode, APIServerHandler} from "@lodestar/api"; import {mapValues} from "@lodestar/utils"; import {BeaconNode} from "../../../src/index.js"; diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index 0be918216b5f..64c223acf4ea 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -1,3 +1,4 @@ +import bls from "@chainsafe/bls"; import {config as minimalConfig} from "@lodestar/config/default"; import { BeaconStateAllForks, @@ -11,7 +12,6 @@ import {allForks, altair, bellatrix, ssz} from "@lodestar/types"; import {createBeaconConfig, ChainForkConfig} from "@lodestar/config"; import {FAR_FUTURE_EPOCH, ForkName, ForkSeq, MAX_EFFECTIVE_BALANCE, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; -import bls from "@chainsafe/bls"; import {generateValidator, generateValidators} from "./validator.js"; import {getConfig} from "./config.js"; diff --git a/packages/beacon-node/test/utils/testnet.ts b/packages/beacon-node/test/utils/testnet.ts index d3b1ceb694aa..0c0c7a8369c8 100644 --- a/packages/beacon-node/test/utils/testnet.ts +++ b/packages/beacon-node/test/utils/testnet.ts @@ -1,7 +1,7 @@ +import {fromHexString} from "@chainsafe/ssz"; import {phase0} from "@lodestar/types"; import {createChainForkConfig, ChainForkConfig} from "@lodestar/config"; import {chainConfig} from "@lodestar/config/default"; -import {fromHexString} from "@chainsafe/ssz"; /** Generic testnet data taken from the Medalla testnet */ export const medallaTestnetConfig = { diff --git a/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts b/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts index b90a8c422e71..45adfec433e4 100644 --- a/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts +++ b/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts @@ -1,8 +1,9 @@ import {computeSigningRoot} from "@lodestar/state-transition"; import {DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_SELECTION_PROOF} from "@lodestar/params"; import {phase0, ssz} from "@lodestar/types"; -import {IBeaconChain} from "../../../src/chain/index.js"; +// eslint-disable-next-line import/no-relative-packages import {getSecretKeyFromIndexCached} from "../../../../state-transition/test/perf/util.js"; +import {IBeaconChain} from "../../../src/chain/index.js"; import {SeenAggregators} from "../../../src/chain/seenCache/index.js"; import {signCached} from "../cache.js"; import {getAttestationValidData, AttestationValidDataOpts} from "./attestation.js"; diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index f27d3c9c020c..314d3d255b62 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -1,17 +1,18 @@ +import {BitArray, toHexString} from "@chainsafe/ssz"; import {computeEpochAtSlot, computeSigningRoot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ProtoBlock, IForkChoice, ExecutionStatus} from "@lodestar/fork-choice"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import {phase0, Slot, ssz} from "@lodestar/types"; -import {BitArray, toHexString} from "@chainsafe/ssz"; import {config} from "@lodestar/config/default"; import {BeaconConfig} from "@lodestar/config"; -import {IBeaconChain} from "../../../src/chain/index.js"; -import {IStateRegenerator} from "../../../src/chain/regen/index.js"; -import {ZERO_HASH, ZERO_HASH_HEX} from "../../../src/constants/index.js"; import { generateTestCachedBeaconStateOnlyValidators, getSecretKeyFromIndexCached, + // eslint-disable-next-line import/no-relative-packages } from "../../../../state-transition/test/perf/util.js"; +import {IBeaconChain} from "../../../src/chain/index.js"; +import {IStateRegenerator} from "../../../src/chain/regen/index.js"; +import {ZERO_HASH, ZERO_HASH_HEX} from "../../../src/constants/index.js"; import {SeenAttesters} from "../../../src/chain/seenCache/index.js"; import {BlsSingleThreadVerifier} from "../../../src/chain/bls/index.js"; import {signCached} from "../cache.js"; diff --git a/packages/cli/package.json b/packages/cli/package.json index e2631096887e..c6fd9f4287e1 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -59,6 +59,9 @@ "@chainsafe/blst": "^0.2.9", "@chainsafe/discv5": "^4.0.0", "@chainsafe/ssz": "^0.10.2", + "@chainsafe/threads": "^1.11.0", + "@libp2p/crypto": "^1.0.0", + "@libp2p/peer-id": "^2.0.3", "@libp2p/peer-id-factory": "^2.0.3", "@lodestar/api": "^1.9.1", "@lodestar/beacon-node": "^1.9.1", diff --git a/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts b/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts index 3dd2c3e4bda9..989aa2f89f63 100644 --- a/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts +++ b/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import os from "node:os"; -import {PeerId} from "@libp2p/interface-peer-id"; +import type {PeerId} from "@libp2p/interface-peer-id"; import {createSecp256k1PeerId} from "@libp2p/peer-id-factory"; import {Multiaddr} from "@multiformats/multiaddr"; import {createKeypairFromPeerId, SignableENR} from "@chainsafe/discv5"; diff --git a/packages/cli/src/cmds/dev/files.ts b/packages/cli/src/cmds/dev/files.ts index 27fa2e650f02..9baf0dc845dd 100644 --- a/packages/cli/src/cmds/dev/files.ts +++ b/packages/cli/src/cmds/dev/files.ts @@ -1,10 +1,10 @@ import fs from "node:fs"; import path from "node:path"; +import {Keystore} from "@chainsafe/bls-keystore"; import {nodeUtils} from "@lodestar/beacon-node"; import {chainConfigToJson, ChainForkConfig} from "@lodestar/config"; import {dumpYaml} from "@lodestar/utils"; import {interopSecretKey} from "@lodestar/state-transition"; -import {Keystore} from "@chainsafe/bls-keystore"; import {PersistedKeysBackend} from "../validator/keymanager/persistedKeys.js"; /* eslint-disable no-console */ diff --git a/packages/cli/src/cmds/lightclient/handler.ts b/packages/cli/src/cmds/lightclient/handler.ts index 030aee032f48..1aaaac5075a1 100644 --- a/packages/cli/src/cmds/lightclient/handler.ts +++ b/packages/cli/src/cmds/lightclient/handler.ts @@ -1,7 +1,7 @@ import path from "node:path"; +import {fromHexString} from "@chainsafe/ssz"; import {ApiError, getClient} from "@lodestar/api"; import {Lightclient} from "@lodestar/light-client"; -import {fromHexString} from "@chainsafe/ssz"; import {LightClientRestTransport} from "@lodestar/light-client/transport"; import {getNodeLogger} from "@lodestar/logger/node"; import {getBeaconConfigFromArgs} from "../../config/beaconParams.js"; diff --git a/packages/cli/src/cmds/validator/blsToExecutionChange.ts b/packages/cli/src/cmds/validator/blsToExecutionChange.ts index 3198aad66f65..ec81a9370bd3 100644 --- a/packages/cli/src/cmds/validator/blsToExecutionChange.ts +++ b/packages/cli/src/cmds/validator/blsToExecutionChange.ts @@ -1,11 +1,11 @@ +import {fromHexString} from "@chainsafe/ssz"; +import bls from "@chainsafe/bls"; +import {PointFormat} from "@chainsafe/bls/types"; import {computeSigningRoot} from "@lodestar/state-transition"; import {DOMAIN_BLS_TO_EXECUTION_CHANGE, ForkName} from "@lodestar/params"; import {createBeaconConfig} from "@lodestar/config"; import {ssz, capella} from "@lodestar/types"; import {ApiError, getClient} from "@lodestar/api"; -import {fromHexString} from "@chainsafe/ssz"; -import bls from "@chainsafe/bls"; -import {PointFormat} from "@chainsafe/bls/types"; import {CliCommand, YargsError} from "../../util/index.js"; import {GlobalArgs} from "../../options/index.js"; diff --git a/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts b/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts index 9121945046d8..e712459bf695 100644 --- a/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts +++ b/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts @@ -1,7 +1,7 @@ import path from "node:path"; +import bls from "@chainsafe/bls"; import {SignerLocal, SignerType} from "@lodestar/validator"; import {LogLevel, Logger} from "@lodestar/utils"; -import bls from "@chainsafe/bls"; import {lockFilepath, unlockFilepath} from "../../../util/lockfile.js"; import {LocalKeystoreDefinition} from "./interface.js"; import {clearKeystoreCache, loadKeystoreCache, writeKeystoreCache} from "./keystoreCache.js"; diff --git a/packages/cli/src/cmds/validator/keymanager/decryptKeystores/threadPool.ts b/packages/cli/src/cmds/validator/keymanager/decryptKeystores/threadPool.ts index e622baa49b66..09f2326a5a2f 100644 --- a/packages/cli/src/cmds/validator/keymanager/decryptKeystores/threadPool.ts +++ b/packages/cli/src/cmds/validator/keymanager/decryptKeystores/threadPool.ts @@ -10,7 +10,10 @@ export class DecryptKeystoresThreadPool { private tasks: QueuedTask, Uint8Array>[] = []; private terminatePoolHandler: () => void; - constructor(keystoreCount: number, private readonly signal: AbortSignal) { + constructor( + keystoreCount: number, + private readonly signal: AbortSignal + ) { this.pool = Pool( () => spawn(new Worker("./worker.js"), { diff --git a/packages/cli/src/cmds/validator/keymanager/impl.ts b/packages/cli/src/cmds/validator/keymanager/impl.ts index f7b91687fb2b..6dabec8ac862 100644 --- a/packages/cli/src/cmds/validator/keymanager/impl.ts +++ b/packages/cli/src/cmds/validator/keymanager/impl.ts @@ -1,5 +1,6 @@ import bls from "@chainsafe/bls"; import {Keystore} from "@chainsafe/bls-keystore"; +import {fromHexString} from "@chainsafe/ssz"; import { Api as KeyManagerClientApi, DeleteRemoteKeyStatus, @@ -12,7 +13,6 @@ import { SignerDefinition, ImportRemoteKeyStatus, } from "@lodestar/api/keymanager"; -import {fromHexString} from "@chainsafe/ssz"; import {Interchange, SignerType, Validator} from "@lodestar/validator"; import {ServerApi} from "@lodestar/api"; import {getPubkeyHexFromKeystore, isValidatePubkeyHex, isValidHttpUrl} from "../../../util/format.js"; diff --git a/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts b/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts index 170f0ab7ca4f..1f1c6cd6e79f 100644 --- a/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts +++ b/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts @@ -2,9 +2,9 @@ import fs from "node:fs"; import path from "node:path"; import bls from "@chainsafe/bls"; import {Keystore} from "@chainsafe/bls-keystore"; +import {PointFormat} from "@chainsafe/bls/types"; import {SignerLocal, SignerType} from "@lodestar/validator"; import {fromHex, toHex} from "@lodestar/utils"; -import {PointFormat} from "@chainsafe/bls/types"; import {writeFile600Perm} from "../../../util/file.js"; import {lockFilepath, unlockFilepath} from "../../../util/lockfile.js"; import {LocalKeystoreDefinition} from "./interface.js"; diff --git a/packages/cli/src/cmds/validator/keymanager/server.ts b/packages/cli/src/cmds/validator/keymanager/server.ts index 8c409d6482e8..dacb32dd600c 100644 --- a/packages/cli/src/cmds/validator/keymanager/server.ts +++ b/packages/cli/src/cmds/validator/keymanager/server.ts @@ -1,8 +1,8 @@ import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import {RestApiServer, RestApiServerOpts, RestApiServerModules} from "@lodestar/beacon-node"; import {toHexString} from "@chainsafe/ssz"; +import {RestApiServer, RestApiServerOpts, RestApiServerModules} from "@lodestar/beacon-node"; import {Api} from "@lodestar/api/keymanager"; import {registerRoutes} from "@lodestar/api/keymanager/server"; import {ChainForkConfig} from "@lodestar/config"; diff --git a/packages/cli/src/cmds/validator/signers/index.ts b/packages/cli/src/cmds/validator/signers/index.ts index a14516e4c86a..57e726a3810f 100644 --- a/packages/cli/src/cmds/validator/signers/index.ts +++ b/packages/cli/src/cmds/validator/signers/index.ts @@ -1,9 +1,9 @@ import path from "node:path"; import bls from "@chainsafe/bls"; import {deriveEth2ValidatorKeys, deriveKeyFromMnemonic} from "@chainsafe/bls-keygen"; +import {toHexString} from "@chainsafe/ssz"; import {interopSecretKey} from "@lodestar/state-transition"; import {externalSignerGetKeys, Signer, SignerType} from "@lodestar/validator"; -import {toHexString} from "@chainsafe/ssz"; import {LogLevel, Logger} from "@lodestar/utils"; import {defaultNetwork, GlobalArgs} from "../../../options/index.js"; import {assertValidPubkeysHex, isValidHttpUrl, parseRange, YargsError} from "../../../util/index.js"; diff --git a/packages/cli/src/config/peerId.ts b/packages/cli/src/config/peerId.ts index 72841b57eca4..bc47f8583831 100644 --- a/packages/cli/src/config/peerId.ts +++ b/packages/cli/src/config/peerId.ts @@ -1,4 +1,4 @@ -import {PeerId} from "@libp2p/interface-peer-id"; +import type {PeerId} from "@libp2p/interface-peer-id"; import {peerIdFromBytes} from "@libp2p/peer-id"; import {createFromPrivKey, createFromPubKey} from "@libp2p/peer-id-factory"; import {unmarshalPrivateKey, unmarshalPublicKey} from "@libp2p/crypto/keys"; diff --git a/packages/cli/test/e2e/blsToExecutionchange.test.ts b/packages/cli/test/e2e/blsToExecutionchange.test.ts index c36fea252db6..9ea73e3b4afd 100644 --- a/packages/cli/test/e2e/blsToExecutionchange.test.ts +++ b/packages/cli/test/e2e/blsToExecutionchange.test.ts @@ -1,9 +1,9 @@ import path from "node:path"; +import {toHexString} from "@chainsafe/ssz"; import {sleep, retry} from "@lodestar/utils"; import {ApiError, getClient} from "@lodestar/api"; import {config} from "@lodestar/config/default"; import {interopSecretKey} from "@lodestar/state-transition"; -import {toHexString} from "@chainsafe/ssz"; import {execCliCommand, spawnCliCommand, stopChildProcess} from "@lodestar/test-utils"; import {testFilesDir} from "../utils.js"; diff --git a/packages/cli/test/unit/cmds/beacon.test.ts b/packages/cli/test/unit/cmds/beacon.test.ts index 8891e3f8d3a8..02aa91f2d4b7 100644 --- a/packages/cli/test/unit/cmds/beacon.test.ts +++ b/packages/cli/test/unit/cmds/beacon.test.ts @@ -3,9 +3,9 @@ import fs from "node:fs"; import {expect} from "chai"; import {createFromJSON, createSecp256k1PeerId} from "@libp2p/peer-id-factory"; import {multiaddr} from "@multiformats/multiaddr"; +import {createKeypairFromPeerId, ENR, SignableENR} from "@chainsafe/discv5"; import {chainConfig} from "@lodestar/config/default"; import {chainConfigToJson} from "@lodestar/config"; -import {createKeypairFromPeerId, ENR, SignableENR} from "@chainsafe/discv5"; import {LogLevel} from "@lodestar/utils"; import {exportToJSON} from "../../../src/config/peerId.js"; import {beaconHandlerInit} from "../../../src/cmds/beacon/handler.js"; diff --git a/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts b/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts index c27d64dcaf2b..59113b273435 100644 --- a/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts +++ b/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts @@ -5,8 +5,8 @@ import {expect} from "chai"; import chainAsPromised from "chai-as-promised"; import chai from "chai"; import {Keystore} from "@chainsafe/bls-keystore"; -import {interopSecretKey} from "@lodestar/state-transition"; import bls from "@chainsafe/bls"; +import {interopSecretKey} from "@lodestar/state-transition"; import {SignerLocal, SignerType} from "@lodestar/validator"; import {loadKeystoreCache, writeKeystoreCache} from "../../../../../src/cmds/validator/keymanager/keystoreCache.js"; import {LocalKeystoreDefinition} from "../../../../../src/cmds/validator/keymanager/interface.js"; diff --git a/packages/cli/test/unit/db.test.ts b/packages/cli/test/unit/db.test.ts index 55f7a3ebaba0..f951b3e6923b 100644 --- a/packages/cli/test/unit/db.test.ts +++ b/packages/cli/test/unit/db.test.ts @@ -1,4 +1,6 @@ +// eslint-disable-next-line import/no-relative-packages import {Bucket as BeaconBucket} from "../../../beacon-node/src/db/buckets.js"; +// eslint-disable-next-line import/no-relative-packages import {Bucket as ValidatorBucket} from "../../../validator/src/buckets.js"; describe("no db bucket overlap", () => { diff --git a/packages/cli/test/utils/mockBeaconApiServer.ts b/packages/cli/test/utils/mockBeaconApiServer.ts index 9ea51d613403..2b4ea14a6e12 100644 --- a/packages/cli/test/utils/mockBeaconApiServer.ts +++ b/packages/cli/test/utils/mockBeaconApiServer.ts @@ -5,6 +5,7 @@ import {ChainForkConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; import {ssz} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; +// eslint-disable-next-line import/no-relative-packages import {testLogger} from "../../../beacon-node/test/utils/logger.js"; const ZERO_HASH_HEX = toHex(Buffer.alloc(32, 0)); diff --git a/packages/cli/test/utils/simulation/assertions/nodeAssertion.ts b/packages/cli/test/utils/simulation/assertions/nodeAssertion.ts index ab2eddb17b23..91ddfca9388d 100644 --- a/packages/cli/test/utils/simulation/assertions/nodeAssertion.ts +++ b/packages/cli/test/utils/simulation/assertions/nodeAssertion.ts @@ -1,5 +1,5 @@ -import {routes} from "@lodestar/api/beacon"; import type {SecretKey} from "@chainsafe/bls/types"; +import {routes} from "@lodestar/api/beacon"; import {ApiError} from "@lodestar/api"; import {AssertionResult, CLClient, CLClientKeys, SimulationAssertion} from "../interfaces.js"; import {arrayEquals} from "../utils/index.js"; diff --git a/packages/cli/test/utils/simulation/interfaces.ts b/packages/cli/test/utils/simulation/interfaces.ts index 7d5af985ea45..f2124ad0edcf 100644 --- a/packages/cli/test/utils/simulation/interfaces.ts +++ b/packages/cli/test/utils/simulation/interfaces.ts @@ -285,7 +285,7 @@ export type StoreTypes[] + Dependencies extends SimulationAssertion[] = SimulationAssertion[], > { readonly id: IdType; capture?( diff --git a/packages/cli/test/utils/simulation/utils/index.ts b/packages/cli/test/utils/simulation/utils/index.ts index f765ec08789d..8f2a32c5b115 100644 --- a/packages/cli/test/utils/simulation/utils/index.ts +++ b/packages/cli/test/utils/simulation/utils/index.ts @@ -80,10 +80,13 @@ export const arrayGroupBy = ( array: T[], predicate: (value: T, index: number, array: T[]) => string ): Record => - array.reduce((acc, value, index, array) => { - (acc[predicate(value, index, array)] ||= []).push(value); - return acc; - }, {} as {[key: string]: T[]}); + array.reduce( + (acc, value, index, array) => { + (acc[predicate(value, index, array)] ||= []).push(value); + return acc; + }, + {} as {[key: string]: T[]} + ); export function strFixedSize(str: string, width: number): string { return str.padEnd(width).slice(0, width); diff --git a/packages/config/src/genesisConfig/index.ts b/packages/config/src/genesisConfig/index.ts index 48d7ee8bf32f..23c9e08d1e8e 100644 --- a/packages/config/src/genesisConfig/index.ts +++ b/packages/config/src/genesisConfig/index.ts @@ -1,6 +1,6 @@ +import {toHexString} from "@chainsafe/ssz"; import {ForkName, SLOTS_PER_EPOCH, DOMAIN_VOLUNTARY_EXIT} from "@lodestar/params"; import {DomainType, ForkDigest, phase0, Root, Slot, ssz, Version} from "@lodestar/types"; -import {toHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "../beaconConfig.js"; import {ForkDigestHex, CachedGenesis} from "./types.js"; export {ForkDigestContext} from "./types.js"; @@ -89,7 +89,7 @@ export function createCachedGenesis(chainForkConfig: ChainForkConfig, genesisVal forkDigest2ForkName(forkDigest: ForkDigest | ForkDigestHex): ForkName { const forkDigestHex = toHexStringNoPrefix(forkDigest); const forkName = forkNameByForkDigest.get(forkDigestHex); - if (!forkName) { + if (forkName == null) { throw Error(`Unknown forkDigest ${forkDigestHex}`); } return forkName; @@ -98,7 +98,7 @@ export function createCachedGenesis(chainForkConfig: ChainForkConfig, genesisVal forkDigest2ForkNameOption(forkDigest: ForkDigest | ForkDigestHex): ForkName | null { const forkDigestHex = toHexStringNoPrefix(forkDigest); const forkName = forkNameByForkDigest.get(forkDigestHex); - if (!forkName) { + if (forkName == null) { return null; } return forkName; diff --git a/packages/config/test/unit/index.test.ts b/packages/config/test/unit/index.test.ts index ac9c416ca309..35dabd5dda61 100644 --- a/packages/config/test/unit/index.test.ts +++ b/packages/config/test/unit/index.test.ts @@ -1,6 +1,6 @@ import {expect} from "chai"; -import {ForkName} from "@lodestar/params"; import {toHexString} from "@chainsafe/ssz"; +import {ForkName} from "@lodestar/params"; import {config, chainConfig} from "../../src/default.js"; import {createForkConfig} from "../../src/index.js"; diff --git a/packages/db/src/abstractRepository.ts b/packages/db/src/abstractRepository.ts index 62f94b3b8bc9..e0462d162ddc 100644 --- a/packages/db/src/abstractRepository.ts +++ b/packages/db/src/abstractRepository.ts @@ -1,5 +1,5 @@ -import {ChainForkConfig} from "@lodestar/config"; import {Type} from "@chainsafe/ssz"; +import {ChainForkConfig} from "@lodestar/config"; import {BUCKET_LENGTH} from "./const.js"; import {FilterOptions, KeyValue} from "./controller/index.js"; import {Db, DbReqOpts} from "./controller/interface.js"; diff --git a/packages/flare/package.json b/packages/flare/package.json index 3e2780c7bae1..2285183a5963 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -58,10 +58,14 @@ "blockchain" ], "dependencies": { + "@chainsafe/bls": "7.1.1", + "@chainsafe/bls-keygen": "^0.3.0", "@lodestar/api": "^1.9.1", "@lodestar/config": "^1.9.1", + "@lodestar/params": "^1.9.1", "@lodestar/state-transition": "^1.9.1", "@lodestar/types": "^1.9.1", + "@lodestar/utils": "^1.9.1", "source-map-support": "^0.5.21", "yargs": "^17.7.1" }, diff --git a/packages/fork-choice/src/forkChoice/store.ts b/packages/fork-choice/src/forkChoice/store.ts index e70aca3ebd16..da831550a584 100644 --- a/packages/fork-choice/src/forkChoice/store.ts +++ b/packages/fork-choice/src/forkChoice/store.ts @@ -1,6 +1,6 @@ +import {toHexString} from "@chainsafe/ssz"; import {EffectiveBalanceIncrements, CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {phase0, Slot, RootHex, ValidatorIndex} from "@lodestar/types"; -import {toHexString} from "@chainsafe/ssz"; import {CheckpointHexWithBalance} from "./interface.js"; /** diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index 54df206b43cd..513d37c393b0 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -1,7 +1,7 @@ +import {toHexString} from "@chainsafe/ssz"; import {Epoch, RootHex, Slot} from "@lodestar/types"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {GENESIS_EPOCH} from "@lodestar/params"; -import {toHexString} from "@chainsafe/ssz"; import {ForkChoiceError, ForkChoiceErrorCode} from "../forkChoice/errors.js"; import {ProtoBlock, ProtoNode, HEX_ZERO_HASH, ExecutionStatus, LVHExecResponse} from "./interface.js"; diff --git a/packages/fork-choice/test/perf/forkChoice/onAttestation.test.ts b/packages/fork-choice/test/perf/forkChoice/onAttestation.test.ts index c2c63369eafa..ae3b10b45d63 100644 --- a/packages/fork-choice/test/perf/forkChoice/onAttestation.test.ts +++ b/packages/fork-choice/test/perf/forkChoice/onAttestation.test.ts @@ -1,9 +1,9 @@ import {itBench} from "@dapplion/benchmark"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {AttestationData, IndexedAttestation} from "@lodestar/types/phase0"; import {ATTESTATION_SUBNET_COUNT} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {computeEpochAtSlot} from "@lodestar/state-transition"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import {initializeForkChoice} from "./util.js"; describe("ForkChoice onAttestation", () => { diff --git a/packages/fork-choice/test/perf/forkChoice/util.ts b/packages/fork-choice/test/perf/forkChoice/util.ts index c8d3472ec3f4..4669b03db6d6 100644 --- a/packages/fork-choice/test/perf/forkChoice/util.ts +++ b/packages/fork-choice/test/perf/forkChoice/util.ts @@ -1,5 +1,5 @@ -import {config} from "@lodestar/config/default"; import {fromHexString} from "@chainsafe/ssz"; +import {config} from "@lodestar/config/default"; import {ExecutionStatus, ForkChoice, IForkChoiceStore, ProtoBlock, ProtoArray} from "../../../src/index.js"; const genesisSlot = 0; diff --git a/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts b/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts index 82f1ff09bf6e..f8f1fef38e15 100644 --- a/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts +++ b/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts @@ -7,6 +7,7 @@ import { getEffectiveBalanceIncrementsZeroed, } from "@lodestar/state-transition"; import {TIMELY_SOURCE_FLAG_INDEX} from "@lodestar/params"; +// eslint-disable-next-line import/no-relative-packages import {generatePerfTestCachedStateAltair} from "../../../../state-transition/test/perf/util.js"; import {VoteTracker} from "../../../src/protoArray/interface.js"; import {computeDeltas} from "../../../src/protoArray/computeDeltas.js"; diff --git a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts index 966d3a27556c..fe4f9a7afaad 100644 --- a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts @@ -1,6 +1,6 @@ import {expect} from "chai"; -import {config} from "@lodestar/config/default"; import {fromHexString} from "@chainsafe/ssz"; +import {config} from "@lodestar/config/default"; import {RootHex, Slot} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; import {computeEpochAtSlot} from "@lodestar/state-transition"; diff --git a/packages/light-client/src/index.ts b/packages/light-client/src/index.ts index a060538874e4..67dc86762ede 100644 --- a/packages/light-client/src/index.ts +++ b/packages/light-client/src/index.ts @@ -1,10 +1,10 @@ import mitt from "mitt"; import {init as initBls} from "@chainsafe/bls/switchable"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@lodestar/params"; import {phase0, RootHex, Slot, SyncPeriod, allForks} from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {isErrorAborted, sleep} from "@lodestar/utils"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; import {getCurrentSlot, slotWithFutureTolerance, timeUntilNextEpoch} from "./utils/clock.js"; import {isNode} from "./utils/utils.js"; import {chunkifyInclusiveRange} from "./utils/chunkify.js"; diff --git a/packages/light-client/src/spec/validateLightClientUpdate.ts b/packages/light-client/src/spec/validateLightClientUpdate.ts index 785f4242a0af..256be6a99c2c 100644 --- a/packages/light-client/src/spec/validateLightClientUpdate.ts +++ b/packages/light-client/src/spec/validateLightClientUpdate.ts @@ -1,7 +1,7 @@ -import {Root, ssz, allForks} from "@lodestar/types"; -import {ChainForkConfig} from "@lodestar/config"; import bls from "@chainsafe/bls/switchable"; import type {PublicKey, Signature} from "@chainsafe/bls/types"; +import {Root, ssz, allForks} from "@lodestar/types"; +import {ChainForkConfig} from "@lodestar/config"; import { FINALIZED_ROOT_INDEX, FINALIZED_ROOT_DEPTH, diff --git a/packages/light-client/src/utils/domain.ts b/packages/light-client/src/utils/domain.ts index ad38a2f714a8..90923ed8c9fc 100644 --- a/packages/light-client/src/utils/domain.ts +++ b/packages/light-client/src/utils/domain.ts @@ -1,7 +1,7 @@ // Only used by processDeposit + lightclient -import {Epoch, Version, Root, DomainType, phase0, ssz, Domain} from "@lodestar/types"; import {Type} from "@chainsafe/ssz"; +import {Epoch, Version, Root, DomainType, phase0, ssz, Domain} from "@lodestar/types"; /** * Return the domain for the [[domainType]] and [[forkVersion]]. diff --git a/packages/light-client/src/utils/utils.ts b/packages/light-client/src/utils/utils.ts index e368879d0866..c6be99bac8ac 100644 --- a/packages/light-client/src/utils/utils.ts +++ b/packages/light-client/src/utils/utils.ts @@ -1,8 +1,8 @@ import bls from "@chainsafe/bls/switchable"; import type {PublicKey} from "@chainsafe/bls/types"; +import {BitArray} from "@chainsafe/ssz"; import {altair, Root, ssz} from "@lodestar/types"; import {BeaconBlockHeader} from "@lodestar/types/phase0"; -import {BitArray} from "@chainsafe/ssz"; import {SyncCommitteeFast} from "../types.js"; export function sumBits(bits: BitArray): number { diff --git a/packages/light-client/src/validation.ts b/packages/light-client/src/validation.ts index d7f44b23148e..a0d6f83d8d02 100644 --- a/packages/light-client/src/validation.ts +++ b/packages/light-client/src/validation.ts @@ -1,6 +1,6 @@ -import {altair, Root, Slot, ssz, allForks} from "@lodestar/types"; import bls from "@chainsafe/bls/switchable"; import type {PublicKey, Signature} from "@chainsafe/bls/types"; +import {altair, Root, Slot, ssz, allForks} from "@lodestar/types"; import { FINALIZED_ROOT_INDEX, FINALIZED_ROOT_DEPTH, diff --git a/packages/light-client/test/unit/isValidLightClientHeader.test.ts b/packages/light-client/test/unit/isValidLightClientHeader.test.ts index b861087748c6..72836fb3169b 100644 --- a/packages/light-client/test/unit/isValidLightClientHeader.test.ts +++ b/packages/light-client/test/unit/isValidLightClientHeader.test.ts @@ -1,7 +1,7 @@ import {expect} from "chai"; +import {fromHexString} from "@chainsafe/ssz"; import {ssz, allForks} from "@lodestar/types"; import {createBeaconConfig, createChainForkConfig, defaultChainConfig} from "@lodestar/config"; -import {fromHexString} from "@chainsafe/ssz"; import {isValidLightClientHeader} from "../../src/spec/utils.js"; describe("isValidLightClientHeader", function () { diff --git a/packages/light-client/test/unit/sync.node.test.ts b/packages/light-client/test/unit/sync.node.test.ts index 316305d3c883..27c924e37462 100644 --- a/packages/light-client/test/unit/sync.node.test.ts +++ b/packages/light-client/test/unit/sync.node.test.ts @@ -1,13 +1,13 @@ import {expect} from "chai"; import {init} from "@chainsafe/bls/switchable"; +import {JsonPath, toHexString} from "@chainsafe/ssz"; +import {computeDescriptor, TreeOffsetProof} from "@chainsafe/persistent-merkle-tree"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params"; import {BeaconStateAllForks, BeaconStateAltair} from "@lodestar/state-transition"; import {altair, ssz} from "@lodestar/types"; import {routes, Api, getClient, ServerApi, ApiError} from "@lodestar/api"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; import {createBeaconConfig, ChainConfig} from "@lodestar/config"; -import {JsonPath, toHexString} from "@chainsafe/ssz"; -import {computeDescriptor, TreeOffsetProof} from "@chainsafe/persistent-merkle-tree"; import {Lightclient, LightclientEvent} from "../../src/index.js"; import {LightclientServerApiMock, ProofServerApiMock} from "../mocks/LightclientServerApiMock.js"; import {EventsServerApiMock} from "../mocks/EventsServerApiMock.js"; diff --git a/packages/light-client/test/utils/prepareUpdateNaive.ts b/packages/light-client/test/utils/prepareUpdateNaive.ts index 2add1f5bf20c..3d0653c97263 100644 --- a/packages/light-client/test/utils/prepareUpdateNaive.ts +++ b/packages/light-client/test/utils/prepareUpdateNaive.ts @@ -1,7 +1,7 @@ -import {altair, Root, ssz} from "@lodestar/types"; import {CompositeViewDU} from "@chainsafe/ssz"; -import {FINALIZED_ROOT_GINDEX, NEXT_SYNC_COMMITTEE_GINDEX, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {Tree} from "@chainsafe/persistent-merkle-tree"; +import {altair, Root, ssz} from "@lodestar/types"; +import {FINALIZED_ROOT_GINDEX, NEXT_SYNC_COMMITTEE_GINDEX, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; export interface IBeaconChainLc { getBlockHeaderByRoot(blockRoot: Root): Promise; diff --git a/packages/logger/src/node.ts b/packages/logger/src/node.ts index f5f847273573..ec2d57b85bbb 100644 --- a/packages/logger/src/node.ts +++ b/packages/logger/src/node.ts @@ -118,7 +118,10 @@ interface DefaultMeta { } export class WinstonLoggerNode extends WinstonLogger implements LoggerNode { - constructor(protected readonly winston: Winston, private readonly opts: LoggerNodeOpts) { + constructor( + protected readonly winston: Winston, + private readonly opts: LoggerNodeOpts + ) { super(winston); } diff --git a/packages/logger/test/unit/logger.test.ts b/packages/logger/test/unit/logger.test.ts index fb063b8e3a17..6cf77fe6a440 100644 --- a/packages/logger/test/unit/logger.test.ts +++ b/packages/logger/test/unit/logger.test.ts @@ -1,5 +1,6 @@ import {expect} from "chai"; import sinon from "sinon"; +// eslint-disable-next-line import/no-relative-packages import {shouldDeleteLogFile} from "../../../cli/src/util/logger.js"; describe("shouldDeleteLogFile", function () { diff --git a/packages/prover/package.json b/packages/prover/package.json index d3122333da49..9dcf674e53b5 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -70,6 +70,8 @@ "@ethereumjs/vm": "^6.4.2", "@lodestar/api": "^1.9.1", "@lodestar/config": "^1.9.1", + "@lodestar/params": "^1.9.1", + "@lodestar/logger": "^1.9.1", "@lodestar/light-client": "^1.9.1", "@lodestar/types": "^1.9.1", "@lodestar/utils": "^1.9.1", @@ -81,7 +83,6 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/logger": "^1.9.1", "@lodestar/test-utils": "^1.9.1", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", diff --git a/packages/prover/test/unit/verified_requests/eth_getBlockByHash.test.ts b/packages/prover/test/unit/verified_requests/eth_getBlockByHash.test.ts index 756e411b927a..71c584a74f1d 100644 --- a/packages/prover/test/unit/verified_requests/eth_getBlockByHash.test.ts +++ b/packages/prover/test/unit/verified_requests/eth_getBlockByHash.test.ts @@ -11,7 +11,7 @@ import {ELBlock} from "../../../src/types.js"; const testCases = [eth_getBlock_with_no_accessList, eth_getBlock_with_contractCreation] as [ TestFixture, - TestFixture + TestFixture, ]; describe("verified_requests / eth_getBlockByHash", () => { diff --git a/packages/prover/test/unit/verified_requests/eth_getBlockByNumber.test.ts b/packages/prover/test/unit/verified_requests/eth_getBlockByNumber.test.ts index 1bbaed52d5bf..a4f1f9d7f869 100644 --- a/packages/prover/test/unit/verified_requests/eth_getBlockByNumber.test.ts +++ b/packages/prover/test/unit/verified_requests/eth_getBlockByNumber.test.ts @@ -11,7 +11,7 @@ import {TestFixture, generateReqHandlerOptionsMock} from "../../mocks/request_ha const testCases = [eth_getBlock_with_no_accessList, eth_getBlock_with_contractCreation] as [ TestFixture, - TestFixture + TestFixture, ]; describe("verified_requests / eth_getBlockByNumber", () => { diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index b36c791acbe8..c6eb68e8710d 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -61,6 +61,7 @@ "@lodestar/params": "^1.9.1", "@lodestar/utils": "^1.9.1", "it-all": "^3.0.2", + "it-pipe": "^3.0.1", "snappy": "^7.2.2", "snappyjs": "^0.7.0", "uint8arraylist": "^2.4.3", diff --git a/packages/reqresp/src/ReqResp.ts b/packages/reqresp/src/ReqResp.ts index 238e3c67f533..b2e1d34ef502 100644 --- a/packages/reqresp/src/ReqResp.ts +++ b/packages/reqresp/src/ReqResp.ts @@ -1,7 +1,7 @@ import {setMaxListeners} from "node:events"; import {Connection, Stream} from "@libp2p/interface-connection"; import {PeerId} from "@libp2p/interface-peer-id"; -import {Libp2p} from "libp2p"; +import type {Libp2p} from "libp2p"; import {Logger} from "@lodestar/utils"; import {getMetrics, Metrics, MetricsRegister} from "./metrics.js"; import {RequestError, RequestErrorCode, sendRequest, SendRequestOpts} from "./request/index.js"; @@ -64,7 +64,10 @@ export class ReqResp { private readonly registeredProtocols = new Map(); private readonly dialOnlyProtocols = new Map(); - constructor(modules: ReqRespProtocolModules, private readonly opts: ReqRespOpts = {}) { + constructor( + modules: ReqRespProtocolModules, + private readonly opts: ReqRespOpts = {} + ) { this.libp2p = modules.libp2p; this.logger = modules.logger; this.metrics = modules.metricsRegister ? getMetrics(modules.metricsRegister) : null; diff --git a/packages/reqresp/src/encoders/requestDecode.ts b/packages/reqresp/src/encoders/requestDecode.ts index 4095e238196d..e91462ab7602 100644 --- a/packages/reqresp/src/encoders/requestDecode.ts +++ b/packages/reqresp/src/encoders/requestDecode.ts @@ -1,4 +1,4 @@ -import {Sink} from "it-stream-types"; +import type {Sink} from "it-stream-types"; import {Uint8ArrayList} from "uint8arraylist"; import {MixedProtocol} from "../types.js"; import {BufferedSource} from "../utils/index.js"; diff --git a/packages/reqresp/src/request/index.ts b/packages/reqresp/src/request/index.ts index 966f96830905..966dfb2b254a 100644 --- a/packages/reqresp/src/request/index.ts +++ b/packages/reqresp/src/request/index.ts @@ -1,6 +1,6 @@ import {pipe} from "it-pipe"; import {PeerId} from "@libp2p/interface-peer-id"; -import {Libp2p} from "libp2p"; +import type {Libp2p} from "libp2p"; import {Uint8ArrayList} from "uint8arraylist"; import {ErrorAborted, Logger, withTimeout, TimeoutError} from "@lodestar/utils"; import {MixedProtocol, ResponseIncoming} from "../types.js"; diff --git a/packages/reqresp/test/fixtures/messages.ts b/packages/reqresp/test/fixtures/messages.ts index bba370b4a685..da71e70500ed 100644 --- a/packages/reqresp/test/fixtures/messages.ts +++ b/packages/reqresp/test/fixtures/messages.ts @@ -1,7 +1,7 @@ +import {fromHexString} from "@chainsafe/ssz"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig} from "@lodestar/config/default"; import {ssz} from "@lodestar/types"; -import {fromHexString} from "@chainsafe/ssz"; import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ResponseIncoming, TypeSizes} from "../../src/types.js"; import {ZERO_HASH} from "../utils/index.js"; diff --git a/packages/reqresp/test/fixtures/protocols.ts b/packages/reqresp/test/fixtures/protocols.ts index 7ba474766777..f20d891781ef 100644 --- a/packages/reqresp/test/fixtures/protocols.ts +++ b/packages/reqresp/test/fixtures/protocols.ts @@ -1,6 +1,6 @@ +import {ContainerType, UintNumberType, ListBasicType, ValueOf} from "@chainsafe/ssz"; import {ssz} from "@lodestar/types"; import {ForkName} from "@lodestar/params"; -import {ContainerType, UintNumberType, ListBasicType, ValueOf} from "@chainsafe/ssz"; import {ContextBytesType, DialOnlyProtocol, Encoding, ProtocolHandler, Protocol} from "../../src/types.js"; import {getEmptyHandler} from "./messages.js"; import {beaconConfig} from "./messages.js"; diff --git a/packages/state-transition/src/block/initiateValidatorExit.ts b/packages/state-transition/src/block/initiateValidatorExit.ts index f202ae5c1368..e34d4dda7002 100644 --- a/packages/state-transition/src/block/initiateValidatorExit.ts +++ b/packages/state-transition/src/block/initiateValidatorExit.ts @@ -1,5 +1,5 @@ -import {FAR_FUTURE_EPOCH} from "@lodestar/params"; import {CompositeViewDU} from "@chainsafe/ssz"; +import {FAR_FUTURE_EPOCH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {CachedBeaconStateAllForks} from "../types.js"; diff --git a/packages/state-transition/src/block/processAttestationPhase0.ts b/packages/state-transition/src/block/processAttestationPhase0.ts index 42b3cfb07e84..6e7ae506dc93 100644 --- a/packages/state-transition/src/block/processAttestationPhase0.ts +++ b/packages/state-transition/src/block/processAttestationPhase0.ts @@ -1,7 +1,7 @@ +import {toHexString} from "@chainsafe/ssz"; import {phase0, ssz} from "@lodestar/types"; import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; -import {toHexString} from "@chainsafe/ssz"; import {computeEpochAtSlot} from "../util/index.js"; import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../types.js"; import {isValidIndexedAttestation} from "./index.js"; diff --git a/packages/state-transition/src/block/processAttestationsAltair.ts b/packages/state-transition/src/block/processAttestationsAltair.ts index feb0d7f36a77..a77a6d1de0bd 100644 --- a/packages/state-transition/src/block/processAttestationsAltair.ts +++ b/packages/state-transition/src/block/processAttestationsAltair.ts @@ -1,5 +1,5 @@ -import {Epoch, phase0} from "@lodestar/types"; import {byteArrayEquals} from "@chainsafe/ssz"; +import {Epoch, phase0} from "@lodestar/types"; import {intSqrt} from "@lodestar/utils"; import { diff --git a/packages/state-transition/src/block/processBlsToExecutionChange.ts b/packages/state-transition/src/block/processBlsToExecutionChange.ts index 28f803c597bf..1cc3706a756f 100644 --- a/packages/state-transition/src/block/processBlsToExecutionChange.ts +++ b/packages/state-transition/src/block/processBlsToExecutionChange.ts @@ -1,7 +1,7 @@ -import {capella} from "@lodestar/types"; -import {BLS_WITHDRAWAL_PREFIX, ETH1_ADDRESS_WITHDRAWAL_PREFIX} from "@lodestar/params"; import {toHexString, byteArrayEquals} from "@chainsafe/ssz"; import {digest} from "@chainsafe/as-sha256"; +import {capella} from "@lodestar/types"; +import {BLS_WITHDRAWAL_PREFIX, ETH1_ADDRESS_WITHDRAWAL_PREFIX} from "@lodestar/params"; import {verifyBlsToExecutionChangeSignature} from "../signatureSets/index.js"; import {CachedBeaconStateCapella} from "../types.js"; diff --git a/packages/state-transition/src/block/processEth1Data.ts b/packages/state-transition/src/block/processEth1Data.ts index 12581d148e1a..3d1927744328 100644 --- a/packages/state-transition/src/block/processEth1Data.ts +++ b/packages/state-transition/src/block/processEth1Data.ts @@ -1,7 +1,7 @@ -import {EPOCHS_PER_ETH1_VOTING_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params"; -import {phase0, ssz} from "@lodestar/types"; import {Node} from "@chainsafe/persistent-merkle-tree"; import {CompositeViewDU} from "@chainsafe/ssz"; +import {EPOCHS_PER_ETH1_VOTING_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {phase0, ssz} from "@lodestar/types"; import {BeaconStateAllForks, CachedBeaconStateAllForks} from "../types.js"; /** diff --git a/packages/state-transition/src/block/processExecutionPayload.ts b/packages/state-transition/src/block/processExecutionPayload.ts index cdc3ed9669cb..56b304cbfc90 100644 --- a/packages/state-transition/src/block/processExecutionPayload.ts +++ b/packages/state-transition/src/block/processExecutionPayload.ts @@ -1,5 +1,5 @@ -import {ssz, allForks, capella, deneb} from "@lodestar/types"; import {toHexString, byteArrayEquals} from "@chainsafe/ssz"; +import {ssz, allForks, capella, deneb} from "@lodestar/types"; import {ForkSeq, MAX_BLOBS_PER_BLOCK} from "@lodestar/params"; import {CachedBeaconStateBellatrix, CachedBeaconStateCapella} from "../types.js"; import {getRandaoMix} from "../util/index.js"; diff --git a/packages/state-transition/src/block/processSyncCommittee.ts b/packages/state-transition/src/block/processSyncCommittee.ts index 3ca8e7a78249..e0bfc318e052 100644 --- a/packages/state-transition/src/block/processSyncCommittee.ts +++ b/packages/state-transition/src/block/processSyncCommittee.ts @@ -1,6 +1,6 @@ +import {byteArrayEquals} from "@chainsafe/ssz"; import {altair, ssz} from "@lodestar/types"; import {DOMAIN_SYNC_COMMITTEE, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; -import {byteArrayEquals} from "@chainsafe/ssz"; import {computeSigningRoot, ISignatureSet, SignatureSetType, verifySignatureSet} from "../util/index.js"; import {CachedBeaconStateAllForks} from "../types.js"; import {G2_POINT_AT_INFINITY} from "../constants/index.js"; diff --git a/packages/state-transition/src/block/processWithdrawals.ts b/packages/state-transition/src/block/processWithdrawals.ts index 19858748c18c..ddea73c27a26 100644 --- a/packages/state-transition/src/block/processWithdrawals.ts +++ b/packages/state-transition/src/block/processWithdrawals.ts @@ -1,10 +1,10 @@ +import {byteArrayEquals, toHexString} from "@chainsafe/ssz"; import {ssz, capella} from "@lodestar/types"; import { MAX_EFFECTIVE_BALANCE, MAX_WITHDRAWALS_PER_PAYLOAD, MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP, } from "@lodestar/params"; -import {byteArrayEquals, toHexString} from "@chainsafe/ssz"; import {CachedBeaconStateCapella} from "../types.js"; import {decreaseBalance, hasEth1WithdrawalCredential, isCapellaPayloadHeader} from "../util/index.js"; diff --git a/packages/state-transition/src/cache/syncCommitteeCache.ts b/packages/state-transition/src/cache/syncCommitteeCache.ts index 88b6f5660daa..908d3f2db176 100644 --- a/packages/state-transition/src/cache/syncCommitteeCache.ts +++ b/packages/state-transition/src/cache/syncCommitteeCache.ts @@ -1,5 +1,5 @@ -import {ssz, ValidatorIndex} from "@lodestar/types"; import {CompositeViewDU, toHexString} from "@chainsafe/ssz"; +import {ssz, ValidatorIndex} from "@lodestar/types"; import {PubkeyIndexMap} from "./pubkeyCache.js"; type SyncComitteeValidatorIndexMap = Map; diff --git a/packages/state-transition/src/cache/types.ts b/packages/state-transition/src/cache/types.ts index 7d63139e3eb6..9d0115cee780 100644 --- a/packages/state-transition/src/cache/types.ts +++ b/packages/state-transition/src/cache/types.ts @@ -1,5 +1,5 @@ -import {ssz} from "@lodestar/types"; import {CompositeViewDU} from "@chainsafe/ssz"; +import {ssz} from "@lodestar/types"; export type BeaconStatePhase0 = CompositeViewDU; export type BeaconStateAltair = CompositeViewDU; diff --git a/packages/state-transition/src/epoch/processJustificationAndFinalization.ts b/packages/state-transition/src/epoch/processJustificationAndFinalization.ts index 13e3eef40b22..8526b7f3a749 100644 --- a/packages/state-transition/src/epoch/processJustificationAndFinalization.ts +++ b/packages/state-transition/src/epoch/processJustificationAndFinalization.ts @@ -1,5 +1,5 @@ -import {GENESIS_EPOCH} from "@lodestar/params"; import {BitArray} from "@chainsafe/ssz"; +import {GENESIS_EPOCH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {computeEpochAtSlot, getBlockRoot} from "../util/index.js"; import {CachedBeaconStateAllForks, EpochTransitionCache} from "../types.js"; diff --git a/packages/state-transition/src/epoch/processPendingAttestations.ts b/packages/state-transition/src/epoch/processPendingAttestations.ts index 9e6ce30bba6c..8f68e9735036 100644 --- a/packages/state-transition/src/epoch/processPendingAttestations.ts +++ b/packages/state-transition/src/epoch/processPendingAttestations.ts @@ -1,5 +1,5 @@ -import {Epoch, phase0} from "@lodestar/types"; import {byteArrayEquals} from "@chainsafe/ssz"; +import {Epoch, phase0} from "@lodestar/types"; import {CachedBeaconStatePhase0} from "../types.js"; import {computeStartSlotAtEpoch, getBlockRootAtSlot, AttesterStatus} from "../util/index.js"; diff --git a/packages/state-transition/src/signatureSets/blsToExecutionChange.ts b/packages/state-transition/src/signatureSets/blsToExecutionChange.ts index cf32d9873adf..f0a50643e4ac 100644 --- a/packages/state-transition/src/signatureSets/blsToExecutionChange.ts +++ b/packages/state-transition/src/signatureSets/blsToExecutionChange.ts @@ -1,8 +1,8 @@ +import bls from "@chainsafe/bls"; +import {CoordType} from "@chainsafe/bls/types"; import {DOMAIN_BLS_TO_EXECUTION_CHANGE, ForkName} from "@lodestar/params"; import {capella, ssz} from "@lodestar/types"; import {BeaconConfig} from "@lodestar/config"; -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/bls/types"; import {computeSigningRoot, ISignatureSet, SignatureSetType, verifySignatureSet} from "../util/index.js"; import {CachedBeaconStateAllForks} from "../types.js"; diff --git a/packages/state-transition/src/slot/index.ts b/packages/state-transition/src/slot/index.ts index 13c85b200622..6c4add1d1230 100644 --- a/packages/state-transition/src/slot/index.ts +++ b/packages/state-transition/src/slot/index.ts @@ -1,5 +1,5 @@ -import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {byteArrayEquals} from "@chainsafe/ssz"; +import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {CachedBeaconStateAllForks} from "../types.js"; import {ZERO_HASH} from "../constants/index.js"; diff --git a/packages/state-transition/src/slot/upgradeStateToAltair.ts b/packages/state-transition/src/slot/upgradeStateToAltair.ts index e75c38341fda..a7af38c8f068 100644 --- a/packages/state-transition/src/slot/upgradeStateToAltair.ts +++ b/packages/state-transition/src/slot/upgradeStateToAltair.ts @@ -1,5 +1,5 @@ -import {ssz} from "@lodestar/types"; import {CompositeViewDU} from "@chainsafe/ssz"; +import {ssz} from "@lodestar/types"; import {CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../types.js"; import {newZeroedArray, RootCache} from "../util/index.js"; import {getNextSyncCommittee} from "../util/syncCommittee.js"; diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index 0bfb264c7ccf..f9e93741fa74 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -1,6 +1,6 @@ +import {toHexString} from "@chainsafe/ssz"; import {allForks, Slot, ssz} from "@lodestar/types"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; -import {toHexString} from "@chainsafe/ssz"; import {BeaconStateTransitionMetrics, onPostStateMetrics, onStateCloneMetrics} from "./metrics.js"; import {beforeProcessEpoch, EpochTransitionCacheOpts} from "./cache/epochTransitionCache.js"; import { diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index 925f2f02400f..1041c33d0eb3 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -1,3 +1,4 @@ +import {CompositeViewDU, ListCompositeType} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import { EFFECTIVE_BALANCE_INCREMENT, @@ -9,7 +10,6 @@ import { } from "@lodestar/params"; import {Bytes32, phase0, Root, ssz, TimeSeconds} from "@lodestar/types"; -import {CompositeViewDU, ListCompositeType} from "@chainsafe/ssz"; import {CachedBeaconStateAllForks, BeaconStateAllForks} from "../types.js"; import {createCachedBeaconState} from "../cache/stateCache.js"; import {EpochCacheImmutableData} from "../cache/epochCache.js"; diff --git a/packages/state-transition/src/util/signingRoot.ts b/packages/state-transition/src/util/signingRoot.ts index c5b20917c63f..d05c8a51053b 100644 --- a/packages/state-transition/src/util/signingRoot.ts +++ b/packages/state-transition/src/util/signingRoot.ts @@ -1,5 +1,5 @@ -import {Domain, phase0, ssz} from "@lodestar/types"; import {Type} from "@chainsafe/ssz"; +import {Domain, phase0, ssz} from "@lodestar/types"; /** * Return the signing root of an object by calculating the root of the object-domain tree. diff --git a/packages/state-transition/src/util/weakSubjectivity.ts b/packages/state-transition/src/util/weakSubjectivity.ts index 6d7f91e6f756..dec0c757d985 100644 --- a/packages/state-transition/src/util/weakSubjectivity.ts +++ b/packages/state-transition/src/util/weakSubjectivity.ts @@ -1,9 +1,9 @@ +import {toHexString} from "@chainsafe/ssz"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {EFFECTIVE_BALANCE_INCREMENT, MAX_DEPOSITS, MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH} from "@lodestar/params"; import {Epoch, Root} from "@lodestar/types"; import {ssz} from "@lodestar/types"; import {Checkpoint} from "@lodestar/types/phase0"; -import {toHexString} from "@chainsafe/ssz"; import {ZERO_HASH} from "../constants/constants.js"; import {CachedBeaconStateAllForks, BeaconStateAllForks} from "../types.js"; import {computeEpochAtSlot, getCurrentEpoch, computeCheckpointEpochAtStateSlot} from "./epoch.js"; diff --git a/packages/state-transition/test/perf/block/util.ts b/packages/state-transition/test/perf/block/util.ts index 86056ba5c5c7..5cbd78e1aa74 100644 --- a/packages/state-transition/test/perf/block/util.ts +++ b/packages/state-transition/test/perf/block/util.ts @@ -1,7 +1,7 @@ -import {altair, phase0, ssz} from "@lodestar/types"; import bls from "@chainsafe/bls"; import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; import {BitArray} from "@chainsafe/ssz"; +import {altair, phase0, ssz} from "@lodestar/types"; import {DOMAIN_DEPOSIT, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; import {config} from "@lodestar/config/default"; import { diff --git a/packages/state-transition/test/perf/misc/aggregationBits.test.ts b/packages/state-transition/test/perf/misc/aggregationBits.test.ts index 19446239a82b..7954f6a06a71 100644 --- a/packages/state-transition/test/perf/misc/aggregationBits.test.ts +++ b/packages/state-transition/test/perf/misc/aggregationBits.test.ts @@ -1,7 +1,7 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {BitArray} from "@chainsafe/ssz"; import {MAX_VALIDATORS_PER_COMMITTEE} from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {BitArray} from "@chainsafe/ssz"; describe("aggregationBits", () => { setBenchOpts({noThreshold: true}); diff --git a/packages/state-transition/test/perf/misc/rootEquals.test.ts b/packages/state-transition/test/perf/misc/rootEquals.test.ts index cfbe046ad5dd..9e39ebe13f89 100644 --- a/packages/state-transition/test/perf/misc/rootEquals.test.ts +++ b/packages/state-transition/test/perf/misc/rootEquals.test.ts @@ -1,6 +1,6 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; -import {ssz} from "@lodestar/types"; import {byteArrayEquals, fromHexString} from "@chainsafe/ssz"; +import {ssz} from "@lodestar/types"; // As of Jun 17 2021 // Compare state root diff --git a/packages/state-transition/test/perf/util.ts b/packages/state-transition/test/perf/util.ts index 72bc2a17ad36..316860fef447 100644 --- a/packages/state-transition/test/perf/util.ts +++ b/packages/state-transition/test/perf/util.ts @@ -1,8 +1,8 @@ -import {config} from "@lodestar/config/default"; -import {allForks, phase0, ssz, Slot, altair} from "@lodestar/types"; import {CoordType, PublicKey, SecretKey} from "@chainsafe/bls/types"; import bls from "@chainsafe/bls"; import {BitArray, fromHexString} from "@chainsafe/ssz"; +import {allForks, phase0, ssz, Slot, altair} from "@lodestar/types"; +import {config} from "@lodestar/config/default"; import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import { EPOCHS_PER_ETH1_VOTING_PERIOD, diff --git a/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts b/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts index bf941a5da65b..9e084dc783a3 100644 --- a/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts +++ b/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts @@ -1,10 +1,10 @@ import crypto from "node:crypto"; import {expect} from "chai"; import bls from "@chainsafe/bls"; +import {BitArray} from "@chainsafe/ssz"; import {config} from "@lodestar/config/default"; import {phase0, capella, ValidatorIndex, BLSSignature, ssz} from "@lodestar/types"; import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; -import {BitArray} from "@chainsafe/ssz"; import {ZERO_HASH} from "../../../src/constants/index.js"; import {getBlockSignatureSets} from "../../../src/signatureSets/index.js"; import {generateCachedState} from "../../utils/state.js"; diff --git a/packages/state-transition/test/unit/util/seed.test.ts b/packages/state-transition/test/unit/util/seed.test.ts index 1b4b28835557..ccdc3332cdba 100644 --- a/packages/state-transition/test/unit/util/seed.test.ts +++ b/packages/state-transition/test/unit/util/seed.test.ts @@ -1,7 +1,7 @@ import {expect} from "chai"; -import {GENESIS_EPOCH, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {toHexString} from "@chainsafe/ssz"; +import {GENESIS_EPOCH, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {getRandaoMix} from "../../../src/util/index.js"; import {generateState} from "../../utils/state.js"; diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 0045378f5d46..061e1313c445 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -45,7 +45,7 @@ "build:watch": "yarn run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", "check-types": "tsc", - "lint": "eslint --color --ext .ts src/ test/", + "lint": "eslint --color --ext .ts src/", "lint:fix": "yarn run lint --fix", "pretest": "yarn run check-types", "check-readme": "typescript-docs-verifier" diff --git a/packages/utils/src/map.ts b/packages/utils/src/map.ts index 3d4f61964e21..15ac012e528c 100644 --- a/packages/utils/src/map.ts +++ b/packages/utils/src/map.ts @@ -19,7 +19,10 @@ export class MapDef extends Map { export class MapDefMax { private readonly map = new Map(); - constructor(private readonly getDefault: () => V, private readonly maxKeys: number) {} + constructor( + private readonly getDefault: () => V, + private readonly maxKeys: number + ) {} getOrDefault(key: K): V { let value = this.map.get(key); diff --git a/packages/validator/src/index.ts b/packages/validator/src/index.ts index 03a124e7f593..a6f3f878d358 100644 --- a/packages/validator/src/index.ts +++ b/packages/validator/src/index.ts @@ -12,8 +12,6 @@ export { } from "./services/validatorStore.js"; export {waitForGenesis} from "./genesis.js"; export {getMetrics, Metrics, MetricsRegister} from "./metrics.js"; -// For CLI to read genesisValidatorsRoot -export {MetaDataRepository} from "./repositories/index.js"; // Remote signer client export { diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 62c89ca7470f..b5bd851760fd 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -1,8 +1,8 @@ +import {toHexString} from "@chainsafe/ssz"; import {BLSSignature, phase0, Slot, ssz} from "@lodestar/types"; import {computeEpochAtSlot, isAggregatorFromCommitteeLength} from "@lodestar/state-transition"; import {sleep} from "@lodestar/utils"; import {Api, ApiError, routes} from "@lodestar/api"; -import {toHexString} from "@chainsafe/ssz"; import {IClock, LoggerVc} from "../util/index.js"; import {PubkeyHex} from "../types.js"; import {Metrics} from "../metrics.js"; diff --git a/packages/validator/src/services/attestationDuties.ts b/packages/validator/src/services/attestationDuties.ts index 8762c4258ce0..0100ca45568e 100644 --- a/packages/validator/src/services/attestationDuties.ts +++ b/packages/validator/src/services/attestationDuties.ts @@ -1,9 +1,9 @@ +import {toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {sleep} from "@lodestar/utils"; import {computeEpochAtSlot, isAggregatorFromCommitteeLength} from "@lodestar/state-transition"; import {BLSSignature, Epoch, Slot, ValidatorIndex, RootHex} from "@lodestar/types"; import {Api, ApiError, routes} from "@lodestar/api"; -import {toHexString} from "@chainsafe/ssz"; import {batchItems, IClock, LoggerVc} from "../util/index.js"; import {PubkeyHex} from "../types.js"; import {Metrics} from "../metrics.js"; diff --git a/packages/validator/src/services/block.ts b/packages/validator/src/services/block.ts index b83c239b62e8..6e8dbe120f60 100644 --- a/packages/validator/src/services/block.ts +++ b/packages/validator/src/services/block.ts @@ -1,3 +1,4 @@ +import {toHexString} from "@chainsafe/ssz"; import { BLSPubkey, Slot, @@ -13,7 +14,6 @@ import { import {ChainForkConfig} from "@lodestar/config"; import {ForkName} from "@lodestar/params"; import {extendError, prettyBytes, racePromisesWithCutoff, RaceEvent} from "@lodestar/utils"; -import {toHexString} from "@chainsafe/ssz"; import { Api, ApiError, diff --git a/packages/validator/src/services/blockDuties.ts b/packages/validator/src/services/blockDuties.ts index 8eae7c40a30d..d0812f173c9e 100644 --- a/packages/validator/src/services/blockDuties.ts +++ b/packages/validator/src/services/blockDuties.ts @@ -1,6 +1,6 @@ +import {toHexString} from "@chainsafe/ssz"; import {computeEpochAtSlot} from "@lodestar/state-transition"; import {BLSPubkey, Epoch, RootHex, Slot} from "@lodestar/types"; -import {toHexString} from "@chainsafe/ssz"; import {Api, ApiError, routes} from "@lodestar/api"; import {IClock, differenceHex, LoggerVc} from "../util/index.js"; import {PubkeyHex} from "../types.js"; diff --git a/packages/validator/src/services/chainHeaderTracker.ts b/packages/validator/src/services/chainHeaderTracker.ts index 6aa0d6fe5940..ed8471721e32 100644 --- a/packages/validator/src/services/chainHeaderTracker.ts +++ b/packages/validator/src/services/chainHeaderTracker.ts @@ -1,8 +1,8 @@ +import {fromHexString} from "@chainsafe/ssz"; import {Api, routes} from "@lodestar/api"; import {Logger} from "@lodestar/utils"; import {Slot, Root, RootHex} from "@lodestar/types"; import {GENESIS_SLOT} from "@lodestar/params"; -import {fromHexString} from "@chainsafe/ssz"; import {ValidatorEvent, ValidatorEventEmitter} from "./emitter.js"; const {EventType} = routes.events; diff --git a/packages/validator/src/services/indices.ts b/packages/validator/src/services/indices.ts index a3c29982bf58..2c8840b44bcd 100644 --- a/packages/validator/src/services/indices.ts +++ b/packages/validator/src/services/indices.ts @@ -1,6 +1,6 @@ +import {toHexString} from "@chainsafe/ssz"; import {ValidatorIndex} from "@lodestar/types"; import {Logger, MapDef} from "@lodestar/utils"; -import {toHexString} from "@chainsafe/ssz"; import {Api, ApiError, routes} from "@lodestar/api"; import {batchItems} from "../util/index.js"; import {Metrics} from "../metrics.js"; @@ -46,7 +46,11 @@ export class IndicesService { // Request indices once private pollValidatorIndicesPromise: Promise | null = null; - constructor(private readonly logger: Logger, private readonly api: Api, private readonly metrics: Metrics | null) { + constructor( + private readonly logger: Logger, + private readonly api: Api, + private readonly metrics: Metrics | null + ) { if (metrics) { metrics.indices.addCollect(() => metrics.indices.set(this.index2pubkey.size)); } diff --git a/packages/validator/src/services/syncCommitteeDuties.ts b/packages/validator/src/services/syncCommitteeDuties.ts index 764cafa0e3e1..e4c3fa2a1bed 100644 --- a/packages/validator/src/services/syncCommitteeDuties.ts +++ b/packages/validator/src/services/syncCommitteeDuties.ts @@ -1,8 +1,8 @@ +import {toHexString} from "@chainsafe/ssz"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params"; import {computeSyncPeriodAtEpoch, computeSyncPeriodAtSlot, isSyncCommitteeAggregator} from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {BLSSignature, Epoch, Slot, SyncPeriod, ValidatorIndex} from "@lodestar/types"; -import {toHexString} from "@chainsafe/ssz"; import {Api, ApiError, routes} from "@lodestar/api"; import {IClock, LoggerVc} from "../util/index.js"; import {PubkeyHex} from "../types.js"; diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 4482eed8727a..2c35a9f345d9 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -1,3 +1,5 @@ +import type {SecretKey} from "@chainsafe/bls/types"; +import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; import { computeEpochAtSlot, computeSigningRoot, @@ -20,7 +22,6 @@ import { DOMAIN_APPLICATION_BUILDER, DOMAIN_BLOB_SIDECAR, } from "@lodestar/params"; -import type {SecretKey} from "@chainsafe/bls/types"; import { allForks, altair, @@ -34,7 +35,6 @@ import { ssz, ValidatorIndex, } from "@lodestar/types"; -import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; import {routes} from "@lodestar/api"; import {ISlashingProtection} from "../slashingProtection/index.js"; import {PubkeyHex} from "../types.js"; diff --git a/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts b/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts index aec41b0cf225..7606d31338c4 100644 --- a/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts +++ b/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts @@ -1,7 +1,7 @@ +import {ContainerType, Type} from "@chainsafe/ssz"; import {BLSPubkey, Epoch, ssz} from "@lodestar/types"; import {intToBytes, bytesToInt} from "@lodestar/utils"; import {DB_PREFIX_LENGTH, DbReqOpts, encodeKey, uintLen} from "@lodestar/db"; -import {ContainerType, Type} from "@chainsafe/ssz"; import {LodestarValidatorDatabaseController} from "../../types.js"; import {SlashingProtectionAttestation} from "../types.js"; import {blsPubkeyLen, uniqueVectorArr} from "../utils.js"; diff --git a/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts b/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts index 30f2f750b240..d45814e6648b 100644 --- a/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts +++ b/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts @@ -1,6 +1,6 @@ +import {ContainerType, Type} from "@chainsafe/ssz"; import {BLSPubkey, Epoch, ssz} from "@lodestar/types"; import {encodeKey, DbReqOpts} from "@lodestar/db"; -import {ContainerType, Type} from "@chainsafe/ssz"; import {LodestarValidatorDatabaseController} from "../../types.js"; import {Bucket, getBucketNameByValue} from "../../buckets.js"; diff --git a/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts b/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts index e7a84589ec0a..25b2fee7e8e5 100644 --- a/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts +++ b/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts @@ -1,7 +1,7 @@ +import {ContainerType, Type} from "@chainsafe/ssz"; import {BLSPubkey, Slot, ssz} from "@lodestar/types"; import {intToBytes, bytesToInt} from "@lodestar/utils"; import {DB_PREFIX_LENGTH, DbReqOpts, encodeKey, uintLen} from "@lodestar/db"; -import {ContainerType, Type} from "@chainsafe/ssz"; import {LodestarValidatorDatabaseController} from "../../types.js"; import {Bucket, getBucketNameByValue} from "../../buckets.js"; import {SlashingProtectionBlock} from "../types.js"; diff --git a/packages/validator/src/slashingProtection/index.ts b/packages/validator/src/slashingProtection/index.ts index 26dc1628307a..505a0981a212 100644 --- a/packages/validator/src/slashingProtection/index.ts +++ b/packages/validator/src/slashingProtection/index.ts @@ -1,5 +1,5 @@ -import {BLSPubkey, Root} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {BLSPubkey, Root} from "@lodestar/types"; import {Logger} from "@lodestar/utils"; import {LodestarValidatorDatabaseController} from "../types.js"; import {uniqueVectorArr} from "../slashingProtection/utils.js"; diff --git a/packages/validator/src/slashingProtection/minMaxSurround/distanceStoreRepository.ts b/packages/validator/src/slashingProtection/minMaxSurround/distanceStoreRepository.ts index f93d1e677d45..db8345a90cc2 100644 --- a/packages/validator/src/slashingProtection/minMaxSurround/distanceStoreRepository.ts +++ b/packages/validator/src/slashingProtection/minMaxSurround/distanceStoreRepository.ts @@ -1,7 +1,7 @@ +import {Type} from "@chainsafe/ssz"; import {encodeKey, DbReqOpts} from "@lodestar/db"; import {BLSPubkey, Epoch, ssz} from "@lodestar/types"; import {intToBytes} from "@lodestar/utils"; -import {Type} from "@chainsafe/ssz"; import {Bucket, getBucketNameByValue} from "../../buckets.js"; import {LodestarValidatorDatabaseController} from "../../types.js"; import {DistanceEntry, IDistanceStore} from "./interface.js"; @@ -26,7 +26,10 @@ class SpanDistanceRepository { private readonly bucketId: string; private readonly dbReqOpts: DbReqOpts; - constructor(protected db: LodestarValidatorDatabaseController, bucket: Bucket) { + constructor( + protected db: LodestarValidatorDatabaseController, + bucket: Bucket + ) { this.type = ssz.Epoch; this.bucket = bucket; this.bucketId = getBucketNameByValue(bucket); diff --git a/packages/validator/src/slashingProtection/utils.ts b/packages/validator/src/slashingProtection/utils.ts index c78b4b990e48..f1fc0f24b9ae 100644 --- a/packages/validator/src/slashingProtection/utils.ts +++ b/packages/validator/src/slashingProtection/utils.ts @@ -1,5 +1,5 @@ -import {Epoch, Root, ssz} from "@lodestar/types"; import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {Epoch, Root, ssz} from "@lodestar/types"; export const blsPubkeyLen = 48; export const ZERO_ROOT = ssz.Root.defaultValue(); diff --git a/packages/validator/src/util/difference.ts b/packages/validator/src/util/difference.ts index 811ac7adf35e..bafc6ff8f42f 100644 --- a/packages/validator/src/util/difference.ts +++ b/packages/validator/src/util/difference.ts @@ -1,5 +1,5 @@ -import {Root} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {Root} from "@lodestar/types"; /** * Return items included in `next` but not in `prev` diff --git a/packages/validator/src/util/externalSignerClient.ts b/packages/validator/src/util/externalSignerClient.ts index bade890c56ab..4bcf66383b05 100644 --- a/packages/validator/src/util/externalSignerClient.ts +++ b/packages/validator/src/util/externalSignerClient.ts @@ -1,11 +1,11 @@ import fetch from "cross-fetch"; +import {ContainerType, toHexString, ValueOf} from "@chainsafe/ssz"; import {phase0, altair, capella} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; import {ValidatorRegistrationV1} from "@lodestar/types/bellatrix"; import {BeaconConfig} from "@lodestar/config"; import {computeEpochAtSlot, blindedOrFullBlockToHeader} from "@lodestar/state-transition"; import {allForks, Epoch, Root, RootHex, Slot, ssz} from "@lodestar/types"; -import {ContainerType, toHexString, ValueOf} from "@chainsafe/ssz"; import {PubkeyHex} from "../types.js"; /* eslint-disable @typescript-eslint/naming-convention */ diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 791cf6fff51f..50d4840694be 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -1,9 +1,9 @@ +import {toHexString} from "@chainsafe/ssz"; import {BLSPubkey, ssz} from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {Genesis} from "@lodestar/types/phase0"; import {Logger} from "@lodestar/utils"; import {getClient, Api, routes, ApiError} from "@lodestar/api"; -import {toHexString} from "@chainsafe/ssz"; import {computeEpochAtSlot, getCurrentSlot} from "@lodestar/state-transition"; import {Clock, IClock} from "./util/clock.js"; import {waitForGenesis} from "./genesis.js"; @@ -67,7 +67,11 @@ export class Validator { private state: Status; private readonly controller: AbortController; - constructor(opts: ValidatorOptions, readonly genesis: Genesis, metrics: Metrics | null = null) { + constructor( + opts: ValidatorOptions, + readonly genesis: Genesis, + metrics: Metrics | null = null + ) { const {db, config: chainConfig, logger, slashingProtection, signers, valProposerConfig} = opts; const config = createBeaconConfig(chainConfig, genesis.genesisValidatorsRoot); this.controller = opts.abortController; diff --git a/packages/validator/test/e2e/web3signer.test.ts b/packages/validator/test/e2e/web3signer.test.ts index 29b65eed1d08..6b852424f2dd 100644 --- a/packages/validator/test/e2e/web3signer.test.ts +++ b/packages/validator/test/e2e/web3signer.test.ts @@ -4,13 +4,13 @@ import tmp from "tmp"; import {expect} from "chai"; import {GenericContainer, Wait, StartedTestContainer} from "testcontainers"; import {Keystore} from "@chainsafe/bls-keystore"; +import bls from "@chainsafe/bls"; import {fromHex, toHex} from "@lodestar/utils"; import {config} from "@lodestar/config/default"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {createBeaconConfig} from "@lodestar/config"; import {genesisData} from "@lodestar/config/networks"; import {getClient, routes} from "@lodestar/api"; -import bls from "@chainsafe/bls"; import {ssz} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; import {Interchange, ISlashingProtection, Signer, SignerType, ValidatorStore} from "../../src/index.js"; diff --git a/packages/validator/test/unit/services/attestationDuties.test.ts b/packages/validator/test/unit/services/attestationDuties.test.ts index 09e2ef70198d..864781b94c74 100644 --- a/packages/validator/test/unit/services/attestationDuties.test.ts +++ b/packages/validator/test/unit/services/attestationDuties.test.ts @@ -1,9 +1,9 @@ import {toBufferBE} from "bigint-buffer"; import {expect} from "chai"; import sinon from "sinon"; -import {chainConfig} from "@lodestar/config/default"; import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; +import {chainConfig} from "@lodestar/config/default"; import {HttpStatusCode, routes} from "@lodestar/api"; import {ssz} from "@lodestar/types"; import {computeEpochAtSlot} from "@lodestar/state-transition"; diff --git a/packages/validator/test/unit/slashingProtection/interchange/index.test.ts b/packages/validator/test/unit/slashingProtection/interchange/index.test.ts index e7e2a5c0bb9b..95137e7d0b71 100644 --- a/packages/validator/test/unit/slashingProtection/interchange/index.test.ts +++ b/packages/validator/test/unit/slashingProtection/interchange/index.test.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {expect} from "chai"; -import {Root, ssz} from "@lodestar/types"; import {toHexString} from "@chainsafe/ssz"; +import {Root, ssz} from "@lodestar/types"; import { Interchange, parseInterchange, diff --git a/packages/validator/test/unit/validatorStore.test.ts b/packages/validator/test/unit/validatorStore.test.ts index 46ac885b16ea..974f2aef87b2 100644 --- a/packages/validator/test/unit/validatorStore.test.ts +++ b/packages/validator/test/unit/validatorStore.test.ts @@ -1,9 +1,9 @@ import {toBufferBE} from "bigint-buffer"; import {expect} from "chai"; import sinon from "sinon"; -import {chainConfig} from "@lodestar/config/default"; import bls from "@chainsafe/bls"; import {toHexString, fromHexString} from "@chainsafe/ssz"; +import {chainConfig} from "@lodestar/config/default"; import {bellatrix} from "@lodestar/types"; import {ValidatorStore} from "../../src/services/validatorStore.js"; diff --git a/packages/validator/test/utils/spec.ts b/packages/validator/test/utils/spec.ts index 3c0ff3b7f5d8..91d6a75d1a52 100644 --- a/packages/validator/test/utils/spec.ts +++ b/packages/validator/test/utils/spec.ts @@ -23,7 +23,7 @@ export type SlashingProtectionInterchangeTest = { target_epoch: string; signing_root?: string; }[]; - } + }, ]; }; diff --git a/yarn.lock b/yarn.lock index 6f8d87567688..2d704a2e0baf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,11 @@ # yarn lockfile v1 +"@aashutoshrathi/word-wrap@^1.2.3": + version "1.2.6" + resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf" + integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA== + "@achingbrain/ip-address@^8.1.0": version "8.1.0" resolved "https://registry.yarnpkg.com/@achingbrain/ip-address/-/ip-address-8.1.0.tgz#24f2e9cd7289e33f433d771b23bea56cfd0242c9" @@ -711,7 +716,7 @@ global-agent "^3.0.0" global-tunnel-ng "^2.7.1" -"@eslint-community/eslint-utils@^4.2.0": +"@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.3.0": version "4.4.0" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA== @@ -723,14 +728,19 @@ resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.5.0.tgz#f6f729b02feee2c749f57e334b7a1b5f40a81724" integrity sha512-vITaYzIcNmjn5tF5uxcZ/ft7/RXGrMUIS9HalWckEOF6ESiwXKoMzAQf2UW0aVd6rnOeExTJVd5hmWXucBKGXQ== -"@eslint/eslintrc@^2.0.3": - version "2.0.3" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.0.3.tgz#4910db5505f4d503f27774bf356e3704818a0331" - integrity sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ== +"@eslint-community/regexpp@^4.5.0": + version "4.5.1" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.5.1.tgz#cdd35dce4fa1a89a4fd42b1599eb35b3af408884" + integrity sha512-Z5ba73P98O1KUYCCJTUeVpja9RcGoMdncZ6T49FCUl2lN38JtCJ+3WgIDBv0AuY4WChU5PmtJmOCTlN6FZTFKQ== + +"@eslint/eslintrc@^2.1.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.1.0.tgz#82256f164cc9e0b59669efc19d57f8092706841d" + integrity sha512-Lj7DECXqIVCqnqjjHMPna4vn6GJcMgul/wuS0je9OZ9gsL0zzDpKPVtcG1HaDVc+9y+qgXneTeUMbCqXJNpH1A== dependencies: ajv "^6.12.4" debug "^4.3.2" - espree "^9.5.2" + espree "^9.6.0" globals "^13.19.0" ignore "^5.2.0" import-fresh "^3.2.1" @@ -738,10 +748,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.43.0": - version "8.43.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.43.0.tgz#559ca3d9ddbd6bf907ad524320a0d14b85586af0" - integrity sha512-s2UHCoiXfxMvmfzqoN+vrQ84ahUSYde9qNO1MdxmoEhyHWsfmwOpFlwYV+ePJEVc7gFnATGUi376WowX1N7tFg== +"@eslint/js@8.44.0": + version "8.44.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.44.0.tgz#961a5903c74139390478bdc808bcde3fc45ab7af" + integrity sha512-Ag+9YM4ocKQx9AarydN0KY2j0ErMHNIocPDrVo8zAE44xLTjEtz81OdR68/cydGtk6m6jDb5Za3r2useMzYmSw== "@ethereumjs/block@^4.2.2": version "4.2.2" @@ -3022,6 +3032,18 @@ node-addon-api "^3.2.1" node-gyp-build "^4.3.0" +"@pkgr/utils@^2.3.1": + version "2.4.2" + resolved "https://registry.yarnpkg.com/@pkgr/utils/-/utils-2.4.2.tgz#9e638bbe9a6a6f165580dc943f138fd3309a2cbc" + integrity sha512-POgTXhjrTfbTV63DiFXav4lBHiICLKKwDeaKn9Nphwj7WH6m0hMMCaJkMyRWjgtPFyRKRVoMXXjczsTQRDEhYw== + dependencies: + cross-spawn "^7.0.3" + fast-glob "^3.3.0" + is-glob "^4.0.3" + open "^9.1.0" + picocolors "^1.0.0" + tslib "^2.6.0" + "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" @@ -3532,11 +3554,16 @@ resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.5.tgz#738dd390a6ecc5442f35e7f03fa1431353f7e138" integrity sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA== -"@types/json-schema@*", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": +"@types/json-schema@*", "@types/json-schema@^7.0.8": version "7.0.11" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== +"@types/json-schema@^7.0.11": + version "7.0.12" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.12.tgz#d70faba7039d5fca54c83c7dbab41051d2b6f6cb" + integrity sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA== + "@types/json5@^0.0.29": version "0.0.29" resolved "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz" @@ -3870,89 +3897,93 @@ dependencies: "@types/node" "*" -"@typescript-eslint/eslint-plugin@5.60.1": - version "5.60.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.60.1.tgz#81382d6ecb92b8dda70e91f9035611cb2fecd1c3" - integrity sha512-KSWsVvsJsLJv3c4e73y/Bzt7OpqMCADUO846bHcuWYSYM19bldbAeDv7dYyV0jwkbMfJ2XdlzwjhXtuD7OY6bw== - dependencies: - "@eslint-community/regexpp" "^4.4.0" - "@typescript-eslint/scope-manager" "5.60.1" - "@typescript-eslint/type-utils" "5.60.1" - "@typescript-eslint/utils" "5.60.1" +"@typescript-eslint/eslint-plugin@6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.0.0.tgz#19ff4f1cab8d6f8c2c1825150f7a840bc5d9bdc4" + integrity sha512-xuv6ghKGoiq856Bww/yVYnXGsKa588kY3M0XK7uUW/3fJNNULKRfZfSBkMTSpqGG/8ZCXCadfh8G/z/B4aqS/A== + dependencies: + "@eslint-community/regexpp" "^4.5.0" + "@typescript-eslint/scope-manager" "6.0.0" + "@typescript-eslint/type-utils" "6.0.0" + "@typescript-eslint/utils" "6.0.0" + "@typescript-eslint/visitor-keys" "6.0.0" debug "^4.3.4" grapheme-splitter "^1.0.4" - ignore "^5.2.0" + graphemer "^1.4.0" + ignore "^5.2.4" + natural-compare "^1.4.0" natural-compare-lite "^1.4.0" - semver "^7.3.7" - tsutils "^3.21.0" + semver "^7.5.0" + ts-api-utils "^1.0.1" -"@typescript-eslint/parser@5.60.1": - version "5.60.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-5.60.1.tgz#0f2f58209c0862a73e3d5a56099abfdfa21d0fd3" - integrity sha512-pHWlc3alg2oSMGwsU/Is8hbm3XFbcrb6P5wIxcQW9NsYBfnrubl/GhVVD/Jm/t8HXhA2WncoIRfBtnCgRGV96Q== +"@typescript-eslint/parser@6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.0.0.tgz#46b2600fd1f67e62fc00a28093a75f41bf7effc4" + integrity sha512-TNaufYSPrr1U8n+3xN+Yp9g31vQDJqhXzzPSHfQDLcaO4tU+mCfODPxCwf4H530zo7aUBE3QIdxCXamEnG04Tg== dependencies: - "@typescript-eslint/scope-manager" "5.60.1" - "@typescript-eslint/types" "5.60.1" - "@typescript-eslint/typescript-estree" "5.60.1" + "@typescript-eslint/scope-manager" "6.0.0" + "@typescript-eslint/types" "6.0.0" + "@typescript-eslint/typescript-estree" "6.0.0" + "@typescript-eslint/visitor-keys" "6.0.0" debug "^4.3.4" -"@typescript-eslint/scope-manager@5.60.1": - version "5.60.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.60.1.tgz#35abdb47f500c68c08f2f2b4f22c7c79472854bb" - integrity sha512-Dn/LnN7fEoRD+KspEOV0xDMynEmR3iSHdgNsarlXNLGGtcUok8L4N71dxUgt3YvlO8si7E+BJ5Fe3wb5yUw7DQ== +"@typescript-eslint/scope-manager@6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.0.0.tgz#8ede47a37cb2b7ed82d329000437abd1113b5e11" + integrity sha512-o4q0KHlgCZTqjuaZ25nw5W57NeykZT9LiMEG4do/ovwvOcPnDO1BI5BQdCsUkjxFyrCL0cSzLjvIMfR9uo7cWg== dependencies: - "@typescript-eslint/types" "5.60.1" - "@typescript-eslint/visitor-keys" "5.60.1" + "@typescript-eslint/types" "6.0.0" + "@typescript-eslint/visitor-keys" "6.0.0" -"@typescript-eslint/type-utils@5.60.1": - version "5.60.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-5.60.1.tgz#17770540e98d65ab4730c7aac618003f702893f4" - integrity sha512-vN6UztYqIu05nu7JqwQGzQKUJctzs3/Hg7E2Yx8rz9J+4LgtIDFWjjl1gm3pycH0P3mHAcEUBd23LVgfrsTR8A== +"@typescript-eslint/type-utils@6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.0.0.tgz#0478d8a94f05e51da2877cc0500f1b3c27ac7e18" + integrity sha512-ah6LJvLgkoZ/pyJ9GAdFkzeuMZ8goV6BH7eC9FPmojrnX9yNCIsfjB+zYcnex28YO3RFvBkV6rMV6WpIqkPvoQ== dependencies: - "@typescript-eslint/typescript-estree" "5.60.1" - "@typescript-eslint/utils" "5.60.1" + "@typescript-eslint/typescript-estree" "6.0.0" + "@typescript-eslint/utils" "6.0.0" debug "^4.3.4" - tsutils "^3.21.0" + ts-api-utils "^1.0.1" -"@typescript-eslint/types@5.60.1": - version "5.60.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.60.1.tgz#a17473910f6b8d388ea83c9d7051af89c4eb7561" - integrity sha512-zDcDx5fccU8BA0IDZc71bAtYIcG9PowaOwaD8rjYbqwK7dpe/UMQl3inJ4UtUK42nOCT41jTSCwg76E62JpMcg== +"@typescript-eslint/types@6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.0.0.tgz#19795f515f8decbec749c448b0b5fc76d82445a1" + integrity sha512-Zk9KDggyZM6tj0AJWYYKgF0yQyrcnievdhG0g5FqyU3Y2DRxJn4yWY21sJC0QKBckbsdKKjYDV2yVrrEvuTgxg== -"@typescript-eslint/typescript-estree@5.60.1": - version "5.60.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.60.1.tgz#8c71824b7165b64d5ebd7aa42968899525959834" - integrity sha512-hkX70J9+2M2ZT6fhti5Q2FoU9zb+GeZK2SLP1WZlvUDqdMbEKhexZODD1WodNRyO8eS+4nScvT0dts8IdaBzfw== +"@typescript-eslint/typescript-estree@6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.0.0.tgz#1e09aab7320e404fb9f83027ea568ac24e372f81" + integrity sha512-2zq4O7P6YCQADfmJ5OTDQTP3ktajnXIRrYAtHM9ofto/CJZV3QfJ89GEaM2BNGeSr1KgmBuLhEkz5FBkS2RQhQ== dependencies: - "@typescript-eslint/types" "5.60.1" - "@typescript-eslint/visitor-keys" "5.60.1" + "@typescript-eslint/types" "6.0.0" + "@typescript-eslint/visitor-keys" "6.0.0" debug "^4.3.4" globby "^11.1.0" is-glob "^4.0.3" - semver "^7.3.7" - tsutils "^3.21.0" + semver "^7.5.0" + ts-api-utils "^1.0.1" -"@typescript-eslint/utils@5.60.1": - version "5.60.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.60.1.tgz#6861ebedbefba1ac85482d2bdef6f2ff1eb65b80" - integrity sha512-tiJ7FFdFQOWssFa3gqb94Ilexyw0JVxj6vBzaSpfN/8IhoKkDuSAenUKvsSHw2A/TMpJb26izIszTXaqygkvpQ== +"@typescript-eslint/utils@6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.0.0.tgz#27a16d0d8f2719274a39417b9782f7daa3802db0" + integrity sha512-SOr6l4NB6HE4H/ktz0JVVWNXqCJTOo/mHnvIte1ZhBQ0Cvd04x5uKZa3zT6tiodL06zf5xxdK8COiDvPnQ27JQ== dependencies: - "@eslint-community/eslint-utils" "^4.2.0" - "@types/json-schema" "^7.0.9" + "@eslint-community/eslint-utils" "^4.3.0" + "@types/json-schema" "^7.0.11" "@types/semver" "^7.3.12" - "@typescript-eslint/scope-manager" "5.60.1" - "@typescript-eslint/types" "5.60.1" - "@typescript-eslint/typescript-estree" "5.60.1" + "@typescript-eslint/scope-manager" "6.0.0" + "@typescript-eslint/types" "6.0.0" + "@typescript-eslint/typescript-estree" "6.0.0" eslint-scope "^5.1.1" - semver "^7.3.7" + semver "^7.5.0" -"@typescript-eslint/visitor-keys@5.60.1": - version "5.60.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.60.1.tgz#19a877358bf96318ec35d90bfe6bd1445cce9434" - integrity sha512-xEYIxKcultP6E/RMKqube11pGjXH1DCo60mQoWhVYyKfLkwbIVVjYxmOenNMxILx0TjCujPTjjnTIVzm09TXIw== +"@typescript-eslint/visitor-keys@6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.0.0.tgz#0b49026049fbd096d2c00c5e784866bc69532a31" + integrity sha512-cvJ63l8c0yXdeT5POHpL0Q1cZoRcmRKFCtSjNGJxPkcP571EfZMcNbzWAc7oK3D1dRzm/V5EwtkANTZxqvuuUA== dependencies: - "@typescript-eslint/types" "5.60.1" - eslint-visitor-keys "^3.3.0" + "@typescript-eslint/types" "6.0.0" + eslint-visitor-keys "^3.4.1" "@webassemblyjs/ast@1.11.6", "@webassemblyjs/ast@^1.11.5": version "1.11.6" @@ -4201,16 +4232,16 @@ acorn@^8.4.1, acorn@^8.7.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== -acorn@^8.8.0: - version "8.8.2" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" - integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== - acorn@^8.8.2: version "8.9.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.9.0.tgz#78a16e3b2bcc198c10822786fa6679e245db5b59" integrity sha512-jaVNAFBHNLXspO543WnNNPZFRtavh3skAkITqD0/2aeMkKZTN+254PyhwxFYrk3vQ1xfY+2wbesJMs/JC8/PwQ== +acorn@^8.9.0: + version "8.10.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.10.0.tgz#8be5b3907a67221a81ab23c7889c4c5526b62ec5" + integrity sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw== + add-stream@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/add-stream/-/add-stream-1.0.0.tgz#6a7990437ca736d5e1288db92bd3266d5f5cb2aa" @@ -4772,6 +4803,11 @@ benchmark@^2.1.4: lodash "^4.17.4" platform "^1.3.3" +big-integer@^1.6.44: + version "1.6.51" + resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.51.tgz#0df92a5d9880560d3ff2d5fd20245c889d130686" + integrity sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg== + bigint-buffer@^1.1.5: version "1.1.5" resolved "https://registry.npmjs.org/bigint-buffer/-/bigint-buffer-1.1.5.tgz" @@ -4945,6 +4981,13 @@ boolean@^3.0.1: resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.2.0.tgz#9e5294af4e98314494cbb17979fa54ca159f116b" integrity sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw== +bplist-parser@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/bplist-parser/-/bplist-parser-0.2.0.tgz#43a9d183e5bf9d545200ceac3e712f79ebbe8d0e" + integrity sha512-z0M+byMThzQmD9NILRniCUXYsYpjwnlO8N5uCFaCqIOpqRsJCrQL9NK3JsD67CN5a08nF5oIL2bD6loTdHOuKw== + dependencies: + big-integer "^1.6.44" + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -5160,6 +5203,13 @@ builtins@^5.0.0: dependencies: semver "^7.0.0" +bundle-name@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/bundle-name/-/bundle-name-3.0.0.tgz#ba59bcc9ac785fb67ccdbf104a2bf60c099f0e1a" + integrity sha512-PKA4BeSvBpQKQ8iPOGCSiell+N8P+Tf1DlwqmYhpe2gAhKPHn8EYOxVT+ShuGmhg8lN8XiSlS80yiExKXrURlw== + dependencies: + run-applescript "^5.0.0" + busboy@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893" @@ -6289,6 +6339,24 @@ deepmerge@^4.3.1: resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== +default-browser-id@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/default-browser-id/-/default-browser-id-3.0.0.tgz#bee7bbbef1f4e75d31f98f4d3f1556a14cea790c" + integrity sha512-OZ1y3y0SqSICtE8DE4S8YOE9UZOJ8wO16fKWVP5J1Qz42kV9jcnMVFrEE/noXb/ss3Q4pZIH79kxofzyNNtUNA== + dependencies: + bplist-parser "^0.2.0" + untildify "^4.0.0" + +default-browser@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/default-browser/-/default-browser-4.0.0.tgz#53c9894f8810bf86696de117a6ce9085a3cbc7da" + integrity sha512-wX5pXO1+BrhMkSbROFsyxUm0i/cJEScyNhA4PPxc41ICuv05ZZB/MX28s8aZx6xjmatvebIapF6hLEKEcpneUA== + dependencies: + bundle-name "^3.0.0" + default-browser-id "^3.0.0" + execa "^7.1.1" + titleize "^3.0.0" + default-gateway@^6.0.2: version "6.0.3" resolved "https://registry.npmjs.org/default-gateway/-/default-gateway-6.0.3.tgz" @@ -6325,6 +6393,11 @@ define-lazy-prop@^2.0.0: resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz#3f7ae421129bcaaac9bc74905c98a0009ec9ee7f" integrity sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og== +define-lazy-prop@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz#dbb19adfb746d7fc6d734a06b72f4a00d021255f" + integrity sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg== + define-properties@^1.1.3, define-properties@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.4.tgz#0b14d7bd7fbeb2f3572c3a7eda80ea5d57fb05b1" @@ -6670,7 +6743,7 @@ enhanced-resolve@^5.0.0: graceful-fs "^4.2.4" tapable "^2.2.0" -enhanced-resolve@^5.15.0: +enhanced-resolve@^5.12.0, enhanced-resolve@^5.15.0: version "5.15.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz#1af946c7d93603eb88e9896cee4904dc012e9c35" integrity sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg== @@ -6943,6 +7016,20 @@ eslint-import-resolver-node@^0.3.7: is-core-module "^2.11.0" resolve "^1.22.1" +eslint-import-resolver-typescript@^3.5.5: + version "3.5.5" + resolved "https://registry.yarnpkg.com/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-3.5.5.tgz#0a9034ae7ed94b254a360fbea89187b60ea7456d" + integrity sha512-TdJqPHs2lW5J9Zpe17DZNQuDnox4xo2o+0tE7Pggain9Rbc19ik8kFtXdxZ250FVx2kF4vlt2RSf4qlUpG7bhw== + dependencies: + debug "^4.3.4" + enhanced-resolve "^5.12.0" + eslint-module-utils "^2.7.4" + get-tsconfig "^4.5.0" + globby "^13.1.3" + is-core-module "^2.11.0" + is-glob "^4.0.3" + synckit "^0.8.5" + eslint-module-utils@^2.7.4: version "2.7.4" resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.7.4.tgz#4f3e41116aaf13a20792261e61d3a2e7e0583974" @@ -6992,12 +7079,13 @@ eslint-plugin-mocha@^10.1.0: eslint-utils "^3.0.0" rambda "^7.1.0" -eslint-plugin-prettier@^4.2.1: - version "4.2.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-4.2.1.tgz#651cbb88b1dab98bfd42f017a12fa6b2d993f94b" - integrity sha512-f/0rXLXUt0oFYs8ra4w49wYZBG5GKZpAYsJSm6rnYL5uVDjd+zowwMwVZHnAjf4edNrKpCDYfXDgmRE/Ak7QyQ== +eslint-plugin-prettier@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-5.0.0.tgz#6887780ed95f7708340ec79acfdf60c35b9be57a" + integrity sha512-AgaZCVuYDXHUGxj/ZGu1u8H8CYgDY3iG6w5kUFw4AzMVXzB7VvbKgYR4nATIN+OvUrghMbiDLeimVjVY5ilq3w== dependencies: prettier-linter-helpers "^1.0.0" + synckit "^0.8.5" eslint-scope@5.1.1, eslint-scope@^5.1.1: version "5.1.1" @@ -7049,15 +7137,15 @@ eslint-visitor-keys@^3.4.1: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz#c22c48f48942d08ca824cc526211ae400478a994" integrity sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA== -eslint@^8.43.0: - version "8.43.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.43.0.tgz#3e8c6066a57097adfd9d390b8fc93075f257a094" - integrity sha512-aaCpf2JqqKesMFGgmRPessmVKjcGXqdlAYLLC3THM8t5nBRZRQ+st5WM/hoJXkdioEXLLbXgclUpM0TXo5HX5Q== +eslint@^8.44.0: + version "8.44.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.44.0.tgz#51246e3889b259bbcd1d7d736a0c10add4f0e500" + integrity sha512-0wpHoUbDUHgNCyvFB5aXLiQVfK9B0at6gUvzy83k4kAsQ/u769TQDX6iKC+aO4upIHO9WSaA3QoXYQDHbNwf1A== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.4.0" - "@eslint/eslintrc" "^2.0.3" - "@eslint/js" "8.43.0" + "@eslint/eslintrc" "^2.1.0" + "@eslint/js" "8.44.0" "@humanwhocodes/config-array" "^0.11.10" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" @@ -7069,7 +7157,7 @@ eslint@^8.43.0: escape-string-regexp "^4.0.0" eslint-scope "^7.2.0" eslint-visitor-keys "^3.4.1" - espree "^9.5.2" + espree "^9.6.0" esquery "^1.4.2" esutils "^2.0.2" fast-deep-equal "^3.1.3" @@ -7089,7 +7177,7 @@ eslint@^8.43.0: lodash.merge "^4.6.2" minimatch "^3.1.2" natural-compare "^1.4.0" - optionator "^0.9.1" + optionator "^0.9.3" strip-ansi "^6.0.1" strip-json-comments "^3.1.0" text-table "^0.2.0" @@ -7099,12 +7187,12 @@ esm@^3.2.25: resolved "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz" integrity sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA== -espree@^9.5.2: - version "9.5.2" - resolved "https://registry.yarnpkg.com/espree/-/espree-9.5.2.tgz#e994e7dc33a082a7a82dceaf12883a829353215b" - integrity sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw== +espree@^9.6.0: + version "9.6.0" + resolved "https://registry.yarnpkg.com/espree/-/espree-9.6.0.tgz#80869754b1c6560f32e3b6929194a3fe07c5b82f" + integrity sha512-1FH/IiruXZ84tpUlm0aCUEwMl2Ho5ilqVh0VvQXw+byAz/4SAciyHLlfmL5WYqsvD38oymdUwBss0LtK8m4s/A== dependencies: - acorn "^8.8.0" + acorn "^8.9.0" acorn-jsx "^5.3.2" eslint-visitor-keys "^3.4.1" @@ -7406,6 +7494,21 @@ execa@^5.0.0: signal-exit "^3.0.3" strip-final-newline "^2.0.0" +execa@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/execa/-/execa-7.1.1.tgz#3eb3c83d239488e7b409d48e8813b76bb55c9c43" + integrity sha512-wH0eMf/UXckdUYnO21+HDztteVv05rq2GXksxT4fCGeHkBhw1DROXh40wcjMcRqDOWE7iPJ4n3M7e2+YFP+76Q== + dependencies: + cross-spawn "^7.0.3" + get-stream "^6.0.1" + human-signals "^4.3.0" + is-stream "^3.0.0" + merge-stream "^2.0.0" + npm-run-path "^5.1.0" + onetime "^6.0.0" + signal-exit "^3.0.7" + strip-final-newline "^3.0.0" + expand-tilde@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz" @@ -7534,6 +7637,17 @@ fast-glob@^3.2.9: merge2 "^1.3.0" micromatch "^4.0.4" +fast-glob@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.0.tgz#7c40cb491e1e2ed5664749e87bfb516dbe8727c0" + integrity sha512-ChDuvbOypPuNjO8yIDf36x7BlZX1smcUMTTcyoIjycexOxd6DFsKsg21qVBzEmr3G7fUKIRy2/psii+CIUt7FA== + dependencies: + "@nodelib/fs.stat" "^2.0.2" + "@nodelib/fs.walk" "^1.2.3" + glob-parent "^5.1.2" + merge2 "^1.3.0" + micromatch "^4.0.4" + fast-json-stable-stringify@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" @@ -8159,6 +8273,13 @@ get-symbol-description@^1.0.0: call-bind "^1.0.2" get-intrinsic "^1.1.1" +get-tsconfig@^4.5.0: + version "4.6.2" + resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.6.2.tgz#831879a5e6c2aa24fe79b60340e2233a1e0f472e" + integrity sha512-E5XrT4CbbXcXWy+1jChlZmrmCwd5KGx502kDCXJJ7y898TtWW9FwoG5HfOLVRKmlmDGkWN2HM9Ho+/Y8F0sJDg== + dependencies: + resolve-pkg-maps "^1.0.0" + getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" @@ -8362,6 +8483,17 @@ globby@11.1.0, globby@^11.0.1, globby@^11.1.0: merge2 "^1.4.1" slash "^3.0.0" +globby@^13.1.3: + version "13.2.2" + resolved "https://registry.yarnpkg.com/globby/-/globby-13.2.2.tgz#63b90b1bf68619c2135475cbd4e71e66aa090592" + integrity sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w== + dependencies: + dir-glob "^3.0.1" + fast-glob "^3.3.0" + ignore "^5.2.4" + merge2 "^1.4.1" + slash "^4.0.0" + gopd@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.0.1.tgz#29ff76de69dac7489b7c0918a5788e56477c332c" @@ -8734,6 +8866,11 @@ human-signals@^2.1.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== +human-signals@^4.3.0: + version "4.3.1" + resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-4.3.1.tgz#ab7f811e851fca97ffbd2c1fe9a958964de321b2" + integrity sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ== + humanize-ms@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" @@ -8803,6 +8940,11 @@ ignore@^5.1.1: resolved "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz" integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== +ignore@^5.2.4: + version "5.2.4" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324" + integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ== + import-fresh@^3.0.0, import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz" @@ -9091,6 +9233,11 @@ is-docker@^2.0.0, is-docker@^2.1.1: resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa" integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ== +is-docker@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-3.0.0.tgz#90093aa3106277d8a77a5910dbae71747e15a200" + integrity sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ== + is-electron@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/is-electron/-/is-electron-2.2.0.tgz" @@ -9135,6 +9282,13 @@ is-hex-prefixed@1.0.0: resolved "https://registry.yarnpkg.com/is-hex-prefixed/-/is-hex-prefixed-1.0.0.tgz#7d8d37e6ad77e5d127148913c573e082d777f554" integrity sha512-WvtOiug1VFrE9v1Cydwm+FnXd3+w9GaeVUss5W4v/SLy3UW00vP+6iNF2SdnfiBoLy4bTqVdkftNGTUeOFVsbA== +is-inside-container@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-inside-container/-/is-inside-container-1.0.0.tgz#e81fba699662eb31dbdaf26766a61d4814717ea4" + integrity sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA== + dependencies: + is-docker "^3.0.0" + is-interactive@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz" @@ -9254,6 +9408,11 @@ is-stream@^2.0.0: resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== +is-stream@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-3.0.0.tgz#e6bfd7aa6bef69f4f472ce9bb681e3e57b4319ac" + integrity sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA== + is-string@^1.0.5, is-string@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.7.tgz#0dd12bf2006f255bb58f695110eff7491eebc0fd" @@ -10663,6 +10822,11 @@ mimic-fn@^2.1.0: resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== +mimic-fn@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-4.0.0.tgz#60a90550d5cb0b239cca65d893b1a53b29871ecc" + integrity sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw== + mimic-response@^1.0.0, mimic-response@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz" @@ -11059,7 +11223,7 @@ multiformats@^11.0.0: resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-11.0.1.tgz#ba58c3f69f032ab67dab4b48cc70f01ac2ca07fe" integrity sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA== -multiformats@^11.0.2: +multiformats@^11.0.1, multiformats@^11.0.2: version "11.0.2" resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-11.0.2.tgz#b14735efc42cd8581e73895e66bebb9752151b60" integrity sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg== @@ -11572,6 +11736,13 @@ npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" +npm-run-path@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-5.1.0.tgz#bc62f7f3f6952d9894bd08944ba011a6ee7b7e00" + integrity sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q== + dependencies: + path-key "^4.0.0" + npmlog@6.0.2, npmlog@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-6.0.2.tgz#c8166017a42f2dea92d6453168dd865186a70830" @@ -11818,6 +11989,13 @@ onetime@^5.1.0, onetime@^5.1.2: dependencies: mimic-fn "^2.1.0" +onetime@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/onetime/-/onetime-6.0.0.tgz#7c24c18ed1fd2e9bca4bd26806a33613c77d34b4" + integrity sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ== + dependencies: + mimic-fn "^4.0.0" + open@^8.4.0: version "8.4.0" resolved "https://registry.yarnpkg.com/open/-/open-8.4.0.tgz#345321ae18f8138f82565a910fdc6b39e8c244f8" @@ -11827,17 +12005,27 @@ open@^8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -optionator@^0.9.1: - version "0.9.1" - resolved "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz" - integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== +open@^9.1.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/open/-/open-9.1.0.tgz#684934359c90ad25742f5a26151970ff8c6c80b6" + integrity sha512-OS+QTnw1/4vrf+9hh1jc1jnYjzSG4ttTBB8UxOwAnInG3Uo4ssetzC1ihqaIHjLJnA5GGlRl6QlZXOTQhRBUvg== dependencies: + default-browser "^4.0.0" + define-lazy-prop "^3.0.0" + is-inside-container "^1.0.0" + is-wsl "^2.2.0" + +optionator@^0.9.3: + version "0.9.3" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.3.tgz#007397d44ed1872fdc6ed31360190f81814e2c64" + integrity sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg== + dependencies: + "@aashutoshrathi/word-wrap" "^1.2.3" deep-is "^0.1.3" fast-levenshtein "^2.0.6" levn "^0.4.1" prelude-ls "^1.2.1" type-check "^0.4.0" - word-wrap "^1.2.3" ora@^5.4.1: version "5.4.1" @@ -12240,6 +12428,11 @@ path-key@^3.0.0, path-key@^3.1.0: resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== +path-key@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-4.0.0.tgz#295588dc3aee64154f877adb9d780b81c554bf18" + integrity sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ== + path-parse@^1.0.5, path-parse@^1.0.6, path-parse@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" @@ -12410,10 +12603,10 @@ prettier-linter-helpers@^1.0.0: dependencies: fast-diff "^1.1.2" -prettier@^2.8.8: - version "2.8.8" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.8.tgz#e8c5d7e98a4305ffe3de2e1fc4aca1a71c28b1da" - integrity sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q== +prettier@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.0.tgz#e7b19f691245a21d618c68bc54dc06122f6105ae" + integrity sha512-zBf5eHpwHOGPC47h0zrPyNn+eAEIdEzfywMoYn2XPi0P44Zp0tSq64rq0xAREh4auw2cJZHo9QUob+NqCQky4g== pretty-format@29.4.3: version "29.4.3" @@ -13028,6 +13221,11 @@ resolve-from@^4.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== +resolve-pkg-maps@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz#616b3dc2c57056b5588c31cdf4b3d64db133720f" + integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw== + resolve-typescript-plugin@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/resolve-typescript-plugin/-/resolve-typescript-plugin-2.0.1.tgz#ae4a1a81372b1e3389239268ac774bcc2780f4e3" @@ -13183,6 +13381,13 @@ roarr@^2.15.3: semver-compare "^1.0.0" sprintf-js "^1.1.2" +run-applescript@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/run-applescript/-/run-applescript-5.0.0.tgz#e11e1c932e055d5c6b40d98374e0268d9b11899c" + integrity sha512-XcT5rBksx1QdIhlFOCtgZkB99ZEouFZ1E2Kc2LHqNW13U3/74YGdkQRmThTwxy4QIyookibDKYZOPqX//6BlAg== + dependencies: + execa "^5.0.0" + run-async@^2.4.0: version "2.4.1" resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455" @@ -13552,6 +13757,11 @@ slash@3.0.0, slash@^3.0.0: resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== +slash@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/slash/-/slash-4.0.0.tgz#2422372176c4c6c5addb5e2ada885af984b396a7" + integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew== + smart-buffer@^4.2.0: version "4.2.0" resolved "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz" @@ -14004,6 +14214,11 @@ strip-final-newline@^2.0.0: resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== +strip-final-newline@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-3.0.0.tgz#52894c313fbff318835280aed60ff71ebf12b8fd" + integrity sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw== + strip-hex-prefix@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/strip-hex-prefix/-/strip-hex-prefix-1.0.0.tgz#0c5f155fef1151373377de9dbb588da05500e36f" @@ -14116,6 +14331,14 @@ swarm-js@^0.1.40: tar "^4.0.2" xhr-request "^1.0.1" +synckit@^0.8.5: + version "0.8.5" + resolved "https://registry.yarnpkg.com/synckit/-/synckit-0.8.5.tgz#b7f4358f9bb559437f9f167eb6bc46b3c9818fa3" + integrity sha512-L1dapNV6vu2s/4Sputv8xGsCdAVlb5nRDMFU/E27D44l5U6cw1g0dGd45uLc+OXjNMmF4ntiMdCimzcjFKQI8Q== + dependencies: + "@pkgr/utils" "^2.3.1" + tslib "^2.5.0" + systeminformation@^5.17.12: version "5.17.12" resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-5.17.12.tgz#5b3e1bfcd5c2c5b459f1a88e61fed27cf9668ba8" @@ -14354,6 +14577,11 @@ tiny-lru@^11.0.1: dependencies: esm "^3.2.25" +titleize@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/titleize/-/titleize-3.0.0.tgz#71c12eb7fdd2558aa8a44b0be83b8a76694acd53" + integrity sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ== + tmp@^0.0.33: version "0.0.33" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" @@ -14448,6 +14676,11 @@ truncate-utf8-bytes@^1.0.0: dependencies: utf8-byte-length "^1.0.1" +ts-api-utils@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.0.1.tgz#8144e811d44c749cd65b2da305a032510774452d" + integrity sha512-lC/RGlPmwdrIBFTX59wwNzqh7aR2otPNPR/5brHZm/XKFYKsfqxihXUe9pU3JI+3vGkl+vyCoNNnPhJn3aLK1A== + ts-loader@^9.4.4: version "9.4.4" resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-9.4.4.tgz#6ceaf4d58dcc6979f84125335904920884b7cee4" @@ -14516,7 +14749,7 @@ tslib@2.5.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.0.tgz#42bfed86f5787aeb41d031866c8f402429e0fddf" integrity sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg== -tslib@^1.10.0, tslib@^1.8.1: +tslib@^1.10.0: version "1.14.1" resolved "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== @@ -14531,18 +14764,16 @@ tslib@^2.1.0: resolved "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz" integrity sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w== +tslib@^2.5.0, tslib@^2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.0.tgz#b295854684dbda164e181d259a22cd779dcd7bc3" + integrity sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA== + tslib@~2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz" integrity sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A== -tsutils@^3.21.0: - version "3.21.0" - resolved "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz" - integrity sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA== - dependencies: - tslib "^1.8.1" - tty-browserify@0.0.0: version "0.0.0" resolved "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz" @@ -14833,6 +15064,11 @@ unpipe@1.0.0, unpipe@~1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +untildify@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" + integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== + upath@2.0.1, upath@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/upath/-/upath-2.0.1.tgz#50c73dea68d6f6b990f51d279ce6081665d61a8b" @@ -15501,11 +15737,6 @@ wipe-webpack-cache@^2.1.0: dependencies: wipe-node-cache "^2.1.0" -word-wrap@^1.2.3: - version "1.2.3" - resolved "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz" - integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== - wordwrap@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" From 3921c2f4257298c015d9a46e49a06284ce3d97a2 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Thu, 13 Jul 2023 12:38:13 +0200 Subject: [PATCH 57/96] chore: publish the @lodestar/prover package (#5744) Publish the @lodestar/prover package --- packages/prover/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/prover/package.json b/packages/prover/package.json index 9dcf674e53b5..490984014d4c 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -1,5 +1,4 @@ { - "private": true, "name": "@lodestar/prover", "description": "A Typescript implementation of the Ethereum Consensus light client", "license": "Apache-2.0", From fbe9beb18d5bb43f0508fe3df97962acf2504f87 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 13 Jul 2023 17:19:42 +0200 Subject: [PATCH 58/96] fix: improve error logging of voluntary-exit command (#5756) * fix: improve error logging of voluntary-exit command * Update packages/cli/src/cmds/validator/voluntaryExit.ts Co-authored-by: Phil Ngo <58080811+philknows@users.noreply.github.com> --------- Co-authored-by: Phil Ngo <58080811+philknows@users.noreply.github.com> --- .../validator/keymanager/decryptKeystoreDefinitions.ts | 4 ++++ packages/cli/src/cmds/validator/voluntaryExit.ts | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts b/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts index e712459bf695..2901fd6cdfb5 100644 --- a/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts +++ b/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts @@ -28,6 +28,10 @@ export async function decryptKeystoreDefinitions( keystoreDefinitions: LocalKeystoreDefinition[], opts: KeystoreDecryptOptions ): Promise { + if (keystoreDefinitions.length === 0) { + return []; + } + if (opts.cacheFilePath) { try { const signers = await loadKeystoreCache(opts.cacheFilePath, keystoreDefinitions); diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index 666003604576..4eb4c5b9a019 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -33,7 +33,7 @@ If no `pubkeys` are provided, it will exit all validators that have been importe examples: [ { - command: "validator voluntary-exit --pubkeys 0xF00", + command: "validator voluntary-exit --network goerli --pubkeys 0xF00", description: "Perform a voluntary exit for the validator who has a public key 0xF00", }, ], @@ -78,6 +78,11 @@ If no `pubkeys` are provided, it will exit all validators that have been importe // Select signers to exit const signers = await getSignersFromArgs(args, network, {logger: console, signal: new AbortController().signal}); + if (signers.length === 0) { + throw new YargsError(`No local keystores found with current args. + Ensure --dataDir and --network match values used when importing keys via validator import + or alternatively, import keys by providing --importKeystores arg to voluntary-exit command.`); + } const signersToExit = selectSignersToExit(args, signers); const validatorsToExit = await resolveValidatorIndexes(client, signersToExit); @@ -153,7 +158,8 @@ async function resolveValidatorIndexes(client: Api, signersToExit: SignerLocalPu return signersToExit.map(({signer, pubkey}) => { const item = dataByPubkey.get(pubkey); if (!item) { - throw Error(`beacon node did not return status for pubkey ${pubkey}`); + throw new YargsError(`Validator with pubkey ${pubkey} is unknown. + Re-check the pubkey submitted or wait until the validator is activated on the beacon chain to voluntary exit.`); } return { From 96820d72b14fdeb4b1c21f6f1c6c8dc604bc8331 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 14 Jul 2023 15:42:36 +0200 Subject: [PATCH 59/96] fix: abort and headers already sent errors for the rest api (#5722) * Fix the validator abort error * Fix abort error * Update the request handling for headers * Update the request handling for headers * Improve the check on the error * Fix prittier error * Revert the changes for the validator handler * Revert the changes for the validator handler --- packages/api/src/beacon/server/events.ts | 11 ++++++++++- packages/cli/src/cmds/validator/signers/index.ts | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/api/src/beacon/server/events.ts b/packages/api/src/beacon/server/events.ts index ed0119dc8f3e..0a20e57e908a 100644 --- a/packages/api/src/beacon/server/events.ts +++ b/packages/api/src/beacon/server/events.ts @@ -1,4 +1,5 @@ import {ChainForkConfig} from "@lodestar/config"; +import {ErrorAborted} from "@lodestar/utils"; import {Api, ReqTypes, routesData, getEventSerdes} from "../routes/events.js"; import {ServerRoutes} from "../../utils/server/index.js"; import {ServerApi} from "../../interfaces.js"; @@ -36,6 +37,9 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR await new Promise((resolve, reject) => { void api.eventstream(req.query.topics, controller.signal, (event) => { try { + // If the request is already aborted, we don't need to send any more events. + if (req.raw.destroyed) return; + const data = eventSerdes.toJson(event); res.raw.write(serializeSSEEvent({event: event.type, data})); } catch (e) { @@ -49,7 +53,12 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR // The client may disconnect and we need to clean the subscriptions. req.raw.once("close", () => resolve()); req.raw.once("end", () => resolve()); - req.raw.once("error", (err) => reject(err)); + req.raw.once("error", (err) => { + if ((err as unknown as {code: string}).code === "ECONNRESET") { + return reject(new ErrorAborted()); + } + return reject(err); + }); }); // api.eventstream will never stop, so no need to ever call `res.raw.end();` diff --git a/packages/cli/src/cmds/validator/signers/index.ts b/packages/cli/src/cmds/validator/signers/index.ts index 57e726a3810f..ea654b8ba221 100644 --- a/packages/cli/src/cmds/validator/signers/index.ts +++ b/packages/cli/src/cmds/validator/signers/index.ts @@ -86,7 +86,7 @@ export async function getSignersFromArgs( const needle = showProgress({ total: keystoreDefinitions.length, frequencyMs: KEYSTORE_IMPORT_PROGRESS_MS, - signal: signal, + signal, progress: ({ratePerSec, percentage, current, total}) => { logger.info( `${percentage.toFixed(0)}% of keystores imported. current=${current} total=${total} rate=${( @@ -119,7 +119,7 @@ export async function getSignersFromArgs( const needle = showProgress({ total: keystoreDefinitions.length, frequencyMs: KEYSTORE_IMPORT_PROGRESS_MS, - signal: signal, + signal, progress: ({ratePerSec, percentage, current, total}) => { logger.info( `${percentage.toFixed(0)}% of local keystores imported. current=${current} total=${total} rate=${( From 34472f488922034091bdf648ce5e1084b988b6b3 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 14 Jul 2023 18:30:42 +0200 Subject: [PATCH 60/96] deps: update @chainsafe/threads to 1.11.1 (#5759) --- packages/beacon-node/package.json | 2 +- packages/cli/package.json | 2 +- yarn.lock | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 753ab35ca5e6..73b37098d887 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -104,7 +104,7 @@ "@chainsafe/persistent-merkle-tree": "^0.5.0", "@chainsafe/prometheus-gc-stats": "^1.0.0", "@chainsafe/ssz": "^0.10.2", - "@chainsafe/threads": "^1.11.0", + "@chainsafe/threads": "^1.11.1", "@ethersproject/abi": "^5.7.0", "@fastify/bearer-auth": "^9.0.0", "@fastify/cors": "^8.2.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index c6fd9f4287e1..af6e9e0434da 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -59,7 +59,7 @@ "@chainsafe/blst": "^0.2.9", "@chainsafe/discv5": "^4.0.0", "@chainsafe/ssz": "^0.10.2", - "@chainsafe/threads": "^1.11.0", + "@chainsafe/threads": "^1.11.1", "@libp2p/crypto": "^1.0.0", "@libp2p/peer-id": "^2.0.3", "@libp2p/peer-id-factory": "^2.0.3", diff --git a/yarn.lock b/yarn.lock index 2d704a2e0baf..0a18d69f71bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -654,10 +654,10 @@ "@chainsafe/as-sha256" "^0.4.1" "@chainsafe/persistent-merkle-tree" "^0.6.1" -"@chainsafe/threads@^1.11.0": - version "1.11.0" - resolved "https://registry.yarnpkg.com/@chainsafe/threads/-/threads-1.11.0.tgz#4845f452d30901053991cf050696746a442a8933" - integrity sha512-l36K9eXqpE0PTnQThDBrxmkx0DX4UV2Qt0l9ASRHRbCUZW3SlSAS+b0//DfTQ2/OXeDVPuE7/72991lTOaKmdQ== +"@chainsafe/threads@^1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@chainsafe/threads/-/threads-1.11.1.tgz#0b3b8c76f5875043ef6d47aeeb681dc80378f205" + integrity sha512-ejkB0eVcM0k2E8n5ZqOGt//ZEWU+c431QS3e/WfjLKxhw/fwZpvYhUOxBA8u3lJmAKLGziIcXjOoTnPwMPhkcQ== dependencies: callsites "^3.1.0" debug "^4.2.0" From 22d7cc88bf1b64a8cb4d726e57c5c313fafecd98 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 14 Jul 2023 20:22:48 +0200 Subject: [PATCH 61/96] feat: support el_offline in eth/v1/node/syncing (#5723) * Add support to check eth1 engine is online or offline * Add e2e tests for elOffline * Fix type for data * Fix types * Update the engine state logic to maintain local state * Update the engine state logic for engine notify * Add logging support for execution engine * Fix logging related tests * Fix lint error * Add event support for execution engine * Add error logging for exception * Update the endpoint test * Use the pyaload status to detect engine state * Fix e2e test for eloffline case * Remove unused variable * Increase wait for CI server * Add dns lookup error code * Update the hardcoded strings to reuse * Remvoe the emitter for execution engine * Update the engine state enum * Update the log prefix for execution * Fix lint error --- packages/api/src/beacon/routes/node.ts | 2 + .../api/test/unit/beacon/testData/node.ts | 2 +- .../src/eth1/provider/jsonRpcHttpClient.ts | 10 ++ .../src/execution/engine/disabled.ts | 6 +- .../beacon-node/src/execution/engine/http.ts | 104 +++++++++++++--- .../src/execution/engine/interface.ts | 10 ++ .../beacon-node/src/execution/engine/utils.ts | 41 ++++++- packages/beacon-node/src/node/nodejs.ts | 7 +- packages/beacon-node/src/sync/sync.ts | 6 + .../api/impl/beacon/node/endpoints.test.ts | 113 ++++++++++++++++++ .../test/sim/merge-interop.test.ts | 2 +- .../test/sim/withdrawal-interop.test.ts | 2 +- .../test/spec/presets/fork_choice.ts | 5 +- .../test/unit/executionEngine/http.test.ts | 3 +- .../unit/executionEngine/httpRetry.test.ts | 3 +- packages/cli/test/sim/endpoints.test.ts | 1 + 16 files changed, 289 insertions(+), 28 deletions(-) create mode 100644 packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts diff --git a/packages/api/src/beacon/routes/node.ts b/packages/api/src/beacon/routes/node.ts index a9476e7e592b..9f1aa8b3cc09 100644 --- a/packages/api/src/beacon/routes/node.ts +++ b/packages/api/src/beacon/routes/node.ts @@ -60,6 +60,8 @@ export type SyncingStatus = { isSyncing: boolean; /** Set to true if the node is optimistically tracking head. */ isOptimistic: boolean; + /** Set to true if the connected el client is offline */ + elOffline: boolean; }; export enum NodeHealth { diff --git a/packages/api/test/unit/beacon/testData/node.ts b/packages/api/test/unit/beacon/testData/node.ts index b8fd36640470..1c1b835edeec 100644 --- a/packages/api/test/unit/beacon/testData/node.ts +++ b/packages/api/test/unit/beacon/testData/node.ts @@ -49,7 +49,7 @@ export const testData: GenericServerTestCases = { }, getSyncingStatus: { args: [], - res: {data: {headSlot: "1", syncDistance: "2", isSyncing: false, isOptimistic: true}}, + res: {data: {headSlot: "1", syncDistance: "2", isSyncing: false, isOptimistic: true, elOffline: false}}, }, getHealth: { args: [], diff --git a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts index cf8a8bf2682b..999147236875 100644 --- a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts +++ b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts @@ -12,6 +12,16 @@ import {encodeJwtToken} from "./jwt.js"; const maxStringLengthToPrint = 500; const REQUEST_TIMEOUT = 30 * 1000; +// As we are using `cross-fetch` which does not support for types for errors +// We can't use `node-fetch` for browser compatibility +export type FetchError = { + errno: string; + code: string; +}; + +export const isFetchError = (error: unknown): error is FetchError => + (error as FetchError) !== undefined && "code" in (error as FetchError) && "errno" in (error as FetchError); + interface RpcResponse extends RpcResponseError { result?: R; } diff --git a/packages/beacon-node/src/execution/engine/disabled.ts b/packages/beacon-node/src/execution/engine/disabled.ts index 6c6804e970c3..82bf84c37d33 100644 --- a/packages/beacon-node/src/execution/engine/disabled.ts +++ b/packages/beacon-node/src/execution/engine/disabled.ts @@ -1,4 +1,4 @@ -import {IExecutionEngine, PayloadIdCache} from "./interface.js"; +import {ExecutionEngineState, IExecutionEngine, PayloadIdCache} from "./interface.js"; export class ExecutionEngineDisabled implements IExecutionEngine { readonly payloadIdCache = new PayloadIdCache(); @@ -26,4 +26,8 @@ export class ExecutionEngineDisabled implements IExecutionEngine { getPayloadBodiesByRange(): Promise { throw Error("Execution engine disabled"); } + + getState(): ExecutionEngineState { + throw Error("Execution engine disabled"); + } } diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index bea26fa95760..60e66aba2d85 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -1,12 +1,13 @@ import {Root, RootHex, allForks, Wei} from "@lodestar/types"; import {SLOTS_PER_EPOCH, ForkName, ForkSeq} from "@lodestar/params"; - +import {Logger} from "@lodestar/logger"; import {ErrorJsonRpcResponse, HttpRpcError} from "../../eth1/provider/jsonRpcHttpClient.js"; import {IJsonRpcHttpClient, ReqOpts} from "../../eth1/provider/jsonRpcHttpClient.js"; import {Metrics} from "../../metrics/index.js"; import {JobItemQueue} from "../../util/queue/index.js"; import {EPOCHS_PER_BATCH} from "../../sync/constants.js"; import {numToQuantity} from "../../eth1/provider/utils.js"; +import {IJson, RpcPayload} from "../../eth1/interface.js"; import { ExecutePayloadStatus, ExecutePayloadResponse, @@ -15,6 +16,7 @@ import { PayloadAttributes, BlobsBundle, VersionedHashes, + ExecutionEngineState, } from "./interface.js"; import {PayloadIdCache} from "./payloadIdCache.js"; import { @@ -29,10 +31,12 @@ import { assertReqSizeLimit, deserializeExecutionPayloadBody, } from "./types.js"; +import {getExecutionEngineState} from "./utils.js"; export type ExecutionEngineModules = { signal: AbortSignal; metrics?: Metrics | null; + logger: Logger; }; export type ExecutionEngineHttpOpts = { @@ -82,6 +86,13 @@ const getPayloadOpts: ReqOpts = {routeId: "getPayload"}; * https://github.com/ethereum/execution-apis/blob/v1.0.0-alpha.1/src/engine/interop/specification.md */ export class ExecutionEngineHttp implements IExecutionEngine { + private logger: Logger; + + // The default state is ONLINE, it will be updated to SYNCING once we receive the first payload + // This assumption is better than the OFFLINE state, since we can't be sure if the EL is offline and being offline may trigger some notifications + // It's safer to to avoid false positives and assume that the EL is syncing until we receive the first payload + private state: ExecutionEngineState = ExecutionEngineState.ONLINE; + readonly payloadIdCache = new PayloadIdCache(); /** * A queue to serialize the fcUs and newPayloads calls: @@ -95,7 +106,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { private readonly rpcFetchQueue: JobItemQueue<[EngineRequest], EngineResponse>; private jobQueueProcessor = async ({method, params, methodOpts}: EngineRequest): Promise => { - return this.rpc.fetchWithRetries( + return this.fetchWithRetries( {method, params}, methodOpts ); @@ -103,13 +114,25 @@ export class ExecutionEngineHttp implements IExecutionEngine { constructor( private readonly rpc: IJsonRpcHttpClient, - {metrics, signal}: ExecutionEngineModules + {metrics, signal, logger}: ExecutionEngineModules ) { this.rpcFetchQueue = new JobItemQueue<[EngineRequest], EngineResponse>( this.jobQueueProcessor, {maxLength: QUEUE_MAX_LENGTH, maxConcurrency: 1, noYieldIfOneItem: true, signal}, metrics?.engineHttpProcessorQueue ); + this.logger = logger; + } + + protected async fetchWithRetries(payload: RpcPayload

, opts?: ReqOpts): Promise { + try { + const res = await this.rpc.fetchWithRetries(payload, opts); + this.updateEngineState(ExecutionEngineState.ONLINE); + return res; + } catch (err) { + this.updateEngineState(getExecutionEngineState({payloadError: err})); + throw err; + } } /** @@ -152,7 +175,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { const serializedExecutionPayload = serializeExecutionPayload(fork, executionPayload); - let engingRequest: EngineRequest; + let engineRequest: EngineRequest; if (ForkSeq[fork] >= ForkSeq.deneb) { if (versionedHashes === undefined) { throw Error(`versionedHashes required in notifyNewPayload for fork=${fork}`); @@ -165,14 +188,14 @@ export class ExecutionEngineHttp implements IExecutionEngine { const parentBeaconBlockRoot = serializeBeaconBlockRoot(parentBlockRoot); const method = "engine_newPayloadV3"; - engingRequest = { + engineRequest = { method, params: [serializedExecutionPayload, serializedVersionedHashes, parentBeaconBlockRoot], methodOpts: notifyNewPayloadOpts, }; } else { const method = ForkSeq[fork] >= ForkSeq.capella ? "engine_newPayloadV2" : "engine_newPayloadV1"; - engingRequest = { + engineRequest = { method, params: [serializedExecutionPayload], methodOpts: notifyNewPayloadOpts, @@ -180,17 +203,19 @@ export class ExecutionEngineHttp implements IExecutionEngine { } const {status, latestValidHash, validationError} = await ( - this.rpcFetchQueue.push(engingRequest) as Promise + this.rpcFetchQueue.push(engineRequest) as Promise ) // If there are errors by EL like connection refused, internal error, they need to be // treated separate from being INVALID. For now, just pass the error upstream. .catch((e: Error): EngineApiRpcReturnTypes[typeof method] => { + this.updateEngineState(getExecutionEngineState({payloadError: e})); if (e instanceof HttpRpcError || e instanceof ErrorJsonRpcResponse) { return {status: ExecutePayloadStatus.ELERROR, latestValidHash: null, validationError: e.message}; } else { return {status: ExecutePayloadStatus.UNAVAILABLE, latestValidHash: null, validationError: e.message}; } }); + this.updateEngineState(getExecutionEngineState({payloadStatus: status})); switch (status) { case ExecutePayloadStatus.VALID: @@ -277,12 +302,21 @@ export class ExecutionEngineHttp implements IExecutionEngine { methodOpts: fcUReqOpts, }) as Promise; - const response = await request; + const response = await request + // If there are errors by EL like connection refused, internal error, they need to be + // treated separate from being INVALID. For now, just pass the error upstream. + .catch((e: Error): EngineApiRpcReturnTypes[typeof method] => { + this.updateEngineState(getExecutionEngineState({payloadError: e})); + throw e; + }); + const { payloadStatus: {status, latestValidHash: _latestValidHash, validationError}, payloadId, } = response; + this.updateEngineState(getExecutionEngineState({payloadStatus: status})); + switch (status) { case ExecutePayloadStatus.VALID: // if payloadAttributes are provided, a valid payloadId is expected @@ -333,7 +367,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { : ForkSeq[fork] >= ForkSeq.capella ? "engine_getPayloadV2" : "engine_getPayloadV1"; - const payloadResponse = await this.rpc.fetchWithRetries< + const payloadResponse = await this.fetchWithRetries< EngineApiRpcReturnTypes[typeof method], EngineApiRpcParamTypes[typeof method] >( @@ -353,13 +387,10 @@ export class ExecutionEngineHttp implements IExecutionEngine { async getPayloadBodiesByHash(blockHashes: RootHex[]): Promise<(ExecutionPayloadBody | null)[]> { const method = "engine_getPayloadBodiesByHashV1"; assertReqSizeLimit(blockHashes.length, 32); - const response = await this.rpc.fetchWithRetries< + const response = await this.fetchWithRetries< EngineApiRpcReturnTypes[typeof method], EngineApiRpcParamTypes[typeof method] - >({ - method, - params: blockHashes, - }); + >({method, params: blockHashes}); return response.map(deserializeExecutionPayloadBody); } @@ -371,15 +402,50 @@ export class ExecutionEngineHttp implements IExecutionEngine { assertReqSizeLimit(blockCount, 32); const start = numToQuantity(startBlockNumber); const count = numToQuantity(blockCount); - const response = await this.rpc.fetchWithRetries< + const response = await this.fetchWithRetries< EngineApiRpcReturnTypes[typeof method], EngineApiRpcParamTypes[typeof method] - >({ - method, - params: [start, count], - }); + >({method, params: [start, count]}); return response.map(deserializeExecutionPayloadBody); } + + getState(): ExecutionEngineState { + return this.state; + } + + private updateEngineState(newState: ExecutionEngineState): void { + const oldState = this.state; + + if (oldState === newState) return; + + // The ONLINE is initial state and can reached from offline or auth failed error + if ( + newState === ExecutionEngineState.ONLINE && + !(oldState === ExecutionEngineState.OFFLINE || oldState === ExecutionEngineState.AUTH_FAILED) + ) { + return; + } + + switch (newState) { + case ExecutionEngineState.ONLINE: + this.logger.info("ExecutionEngine became online"); + break; + case ExecutionEngineState.OFFLINE: + this.logger.error("ExecutionEngine went offline"); + break; + case ExecutionEngineState.SYNCED: + this.logger.info("ExecutionEngine is synced"); + break; + case ExecutionEngineState.SYNCING: + this.logger.info("ExecutionEngine is syncing"); + break; + case ExecutionEngineState.AUTH_FAILED: + this.logger.error("ExecutionEngine authentication failed"); + break; + } + + this.state = newState; + } } type EngineRequestKey = keyof EngineApiRpcParamTypes; diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index 0b2d96229ced..92eff776cacb 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -30,6 +30,14 @@ export enum ExecutePayloadStatus { UNSAFE_OPTIMISTIC_STATUS = "UNSAFE_OPTIMISTIC_STATUS", } +export enum ExecutionEngineState { + ONLINE = "ONLINE", + OFFLINE = "OFFLINE", + SYNCING = "SYNCING", + SYNCED = "SYNCED", + AUTH_FAILED = "AUTH_FAILED", +} + export type ExecutePayloadResponse = | {status: ExecutePayloadStatus.SYNCING | ExecutePayloadStatus.ACCEPTED; latestValidHash: null; validationError: null} | {status: ExecutePayloadStatus.VALID; latestValidHash: RootHex; validationError: null} @@ -132,4 +140,6 @@ export interface IExecutionEngine { getPayloadBodiesByHash(blockHash: DATA[]): Promise<(ExecutionPayloadBody | null)[]>; getPayloadBodiesByRange(start: number, count: number): Promise<(ExecutionPayloadBody | null)[]>; + + getState(): ExecutionEngineState; } diff --git a/packages/beacon-node/src/execution/engine/utils.ts b/packages/beacon-node/src/execution/engine/utils.ts index cbdd51930ba3..1566c1d59984 100644 --- a/packages/beacon-node/src/execution/engine/utils.ts +++ b/packages/beacon-node/src/execution/engine/utils.ts @@ -1,5 +1,6 @@ import {IJson, RpcPayload} from "../../eth1/interface.js"; -import {IJsonRpcHttpClient} from "../../eth1/provider/jsonRpcHttpClient.js"; +import {IJsonRpcHttpClient, isFetchError} from "../../eth1/provider/jsonRpcHttpClient.js"; +import {ExecutePayloadStatus, ExecutionEngineState} from "./interface.js"; export type JsonRpcBackend = { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -27,3 +28,41 @@ export class ExecutionEngineMockJsonRpcClient implements IJsonRpcHttpClient { return Promise.all(rpcPayloadArr.map((payload) => this.fetch(payload))); } } + +const fatalErrorCodes = ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN"]; +const connectionErrorCodes = ["ECONNRESET", "ECONNABORTED"]; + +export function getExecutionEngineState({ + payloadError, + payloadStatus, +}: + | {payloadStatus: ExecutePayloadStatus; payloadError?: never} + | {payloadStatus?: never; payloadError: unknown}): ExecutionEngineState { + switch (payloadStatus) { + case ExecutePayloadStatus.ACCEPTED: + case ExecutePayloadStatus.VALID: + case ExecutePayloadStatus.UNSAFE_OPTIMISTIC_STATUS: + return ExecutionEngineState.SYNCED; + + case ExecutePayloadStatus.ELERROR: + case ExecutePayloadStatus.INVALID: + case ExecutePayloadStatus.SYNCING: + case ExecutePayloadStatus.INVALID_BLOCK_HASH: + return ExecutionEngineState.SYNCING; + + case ExecutePayloadStatus.UNAVAILABLE: + return ExecutionEngineState.OFFLINE; + } + + if (payloadError && isFetchError(payloadError) && fatalErrorCodes.includes(payloadError.code)) { + return ExecutionEngineState.OFFLINE; + } + + if (payloadError && isFetchError(payloadError) && connectionErrorCodes.includes(payloadError.code)) { + return ExecutionEngineState.AUTH_FAILED; + } + + // In case we can't determine the state, we assume it's online + // This assumption is better than considering offline, because the offline state may trigger some notifications + return ExecutionEngineState.ONLINE; +} diff --git a/packages/beacon-node/src/node/nodejs.ts b/packages/beacon-node/src/node/nodejs.ts index 4c8ecd4a7f96..e0b71c36aa94 100644 --- a/packages/beacon-node/src/node/nodejs.ts +++ b/packages/beacon-node/src/node/nodejs.ts @@ -66,6 +66,7 @@ enum LoggerModule { backfill = "backfill", chain = "chain", eth1 = "eth1", + execution = "execution", metrics = "metrics", monitoring = "monitoring", network = "network", @@ -209,7 +210,11 @@ export class BeaconNode { logger: logger.child({module: LoggerModule.eth1}), signal, }), - executionEngine: initializeExecutionEngine(opts.executionEngine, {metrics, signal}), + executionEngine: initializeExecutionEngine(opts.executionEngine, { + metrics, + signal, + logger: logger.child({module: LoggerModule.execution}), + }), executionBuilder: opts.executionBuilder.enabled ? initializeExecutionBuilder(opts.executionBuilder, config, metrics) : undefined, diff --git a/packages/beacon-node/src/sync/sync.ts b/packages/beacon-node/src/sync/sync.ts index 914d511eb8e0..dac8a501cea0 100644 --- a/packages/beacon-node/src/sync/sync.ts +++ b/packages/beacon-node/src/sync/sync.ts @@ -7,6 +7,7 @@ import {Metrics} from "../metrics/index.js"; import {IBeaconChain} from "../chain/index.js"; import {ClockEvent} from "../util/clock.js"; import {GENESIS_SLOT} from "../constants/constants.js"; +import {ExecutionEngineState} from "../execution/index.js"; import {IBeaconSync, SyncModules, SyncingStatus} from "./interface.js"; import {RangeSync, RangeSyncStatus, RangeSyncEvent} from "./range/range.js"; import {getPeerSyncType, PeerSyncType, peerSyncTypes} from "./utils/remoteSyncType.js"; @@ -80,6 +81,8 @@ export class BeaconSync implements IBeaconSync { getSyncStatus(): SyncingStatus { const currentSlot = this.chain.clock.currentSlot; + const elOffline = this.chain.executionEngine.getState() === ExecutionEngineState.OFFLINE; + // If we are pre/at genesis, signal ready if (currentSlot <= GENESIS_SLOT) { return { @@ -87,6 +90,7 @@ export class BeaconSync implements IBeaconSync { syncDistance: "0", isSyncing: false, isOptimistic: false, + elOffline, }; } else { const head = this.chain.forkChoice.getHead(); @@ -100,6 +104,7 @@ export class BeaconSync implements IBeaconSync { syncDistance: String(currentSlot - head.slot), isSyncing: true, isOptimistic: isOptimisticBlock(head), + elOffline, }; case SyncState.Synced: return { @@ -107,6 +112,7 @@ export class BeaconSync implements IBeaconSync { syncDistance: "0", isSyncing: false, isOptimistic: isOptimisticBlock(head), + elOffline, }; default: throw new Error("Node is stopped, cannot get sync status"); diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts new file mode 100644 index 000000000000..ae19523b8a03 --- /dev/null +++ b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts @@ -0,0 +1,113 @@ +import {expect} from "chai"; +import {createBeaconConfig} from "@lodestar/config"; +import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {Api, getClient} from "@lodestar/api/beacon"; +import {ApiError} from "@lodestar/api"; +import {sleep} from "@lodestar/utils"; +import {LogLevel, testLogger} from "../../../../../utils/logger.js"; +import {getDevBeaconNode} from "../../../../../utils/node/beacon.js"; +import {BeaconNode} from "../../../../../../src/node/nodejs.js"; +import {getAndInitDevValidators} from "../../../../../utils/node/validator.js"; + +describe("beacon node api", function () { + this.timeout("30s"); + + const restPort = 9596; + const config = createBeaconConfig(chainConfigDef, Buffer.alloc(32, 0xaa)); + const validatorCount = 8; + + let bn: BeaconNode; + let client: Api; + + before(async () => { + bn = await getDevBeaconNode({ + params: chainConfigDef, + options: { + sync: {isSingleNode: true}, + network: {allowPublishToZeroPeers: true}, + api: { + rest: { + enabled: true, + port: restPort, + }, + }, + chain: {blsVerifyAllMainThread: true}, + }, + validatorCount, + logger: testLogger("Node-A", {level: LogLevel.info}), + }); + client = getClient({baseUrl: `http://127.0.0.1:${restPort}`}, {config}); + }); + + after(async () => { + await bn.close(); + }); + + describe("getSyncingStatus", () => { + it("should return valid syncing status", async () => { + const res = await client.node.getSyncingStatus(); + ApiError.assert(res); + + expect(res.response.data).to.eql({ + headSlot: "0", + syncDistance: "0", + isSyncing: false, + isOptimistic: false, + elOffline: false, + }); + }); + + it("should return 'el_offline' as 'true' for default dev node", async () => { + const res = await client.node.getSyncingStatus(); + ApiError.assert(res); + + expect(res.response.data.elOffline).to.eql(false); + }); + + it("should return 'el_offline' as 'true' when EL not available", async () => { + // Close first instance + await bn.close(); + bn = await getDevBeaconNode({ + params: { + ...chainConfigDef, + // eslint-disable-next-line @typescript-eslint/naming-convention + ALTAIR_FORK_EPOCH: 0, + // eslint-disable-next-line @typescript-eslint/naming-convention + BELLATRIX_FORK_EPOCH: 0, + }, + options: { + sync: {isSingleNode: true}, + network: {allowPublishToZeroPeers: true}, + executionEngine: {mode: "http", urls: ["http://not-available-engine:9999"]}, + api: { + rest: { + enabled: true, + port: restPort, + }, + }, + chain: {blsVerifyAllMainThread: true}, + }, + validatorCount: 5, + logger: testLogger("Node-A", {level: LogLevel.info}), + }); + + // To make BN communicate with EL, it needs to produce some blocks and for that need validators + const {validators} = await getAndInitDevValidators({ + node: bn, + validatorClientCount: 1, + validatorsPerClient: validatorCount, + startIndex: 0, + }); + + // Give node sometime to communicate with EL + await sleep(chainConfigDef.SECONDS_PER_SLOT * 2 * 1000); + + const res = await client.node.getSyncingStatus(); + ApiError.assert(res); + + expect(res.response.data.elOffline).to.eql(true); + + await Promise.all(validators.map((v) => v.close())); + }); + }); +}); diff --git a/packages/beacon-node/test/sim/merge-interop.test.ts b/packages/beacon-node/test/sim/merge-interop.test.ts index 5e079e204a9f..4a57b9308430 100644 --- a/packages/beacon-node/test/sim/merge-interop.test.ts +++ b/packages/beacon-node/test/sim/merge-interop.test.ts @@ -111,7 +111,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { //const controller = new AbortController(); const executionEngine = initializeExecutionEngine( {mode: "http", urls: [engineRpcUrl], jwtSecretHex, retryAttempts, retryDelay}, - {signal: controller.signal} + {signal: controller.signal, logger: testLogger("Node-A-Engine")} ); // 1. Prepare a payload diff --git a/packages/beacon-node/test/sim/withdrawal-interop.test.ts b/packages/beacon-node/test/sim/withdrawal-interop.test.ts index e347adeceea3..29bf5a4e315a 100644 --- a/packages/beacon-node/test/sim/withdrawal-interop.test.ts +++ b/packages/beacon-node/test/sim/withdrawal-interop.test.ts @@ -83,7 +83,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { //const controller = new AbortController(); const executionEngine = initializeExecutionEngine( {mode: "http", urls: [engineRpcUrl], jwtSecretHex, retryAttempts, retryDelay}, - {signal: controller.signal} + {signal: controller.signal, logger: testLogger("executionEngine")} ); const withdrawalsVector = [ diff --git a/packages/beacon-node/test/spec/presets/fork_choice.ts b/packages/beacon-node/test/spec/presets/fork_choice.ts index dae951906811..c98d4419499a 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.ts @@ -65,7 +65,10 @@ export const forkChoiceTest = }); const controller = new AbortController(); - const executionEngine = getExecutionEngineFromBackend(executionEngineBackend, {signal: controller.signal}); + const executionEngine = getExecutionEngineFromBackend(executionEngineBackend, { + signal: controller.signal, + logger: testLogger("executionEngine"), + }); const chain = new BeaconChain( { diff --git a/packages/beacon-node/test/unit/executionEngine/http.test.ts b/packages/beacon-node/test/unit/executionEngine/http.test.ts index 5d82b1f3a230..fc3db1932d8f 100644 --- a/packages/beacon-node/test/unit/executionEngine/http.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/http.test.ts @@ -1,6 +1,7 @@ import {expect} from "chai"; import {fastify} from "fastify"; import {ForkName} from "@lodestar/params"; +import {Logger} from "@lodestar/logger"; import {defaultExecutionEngineHttpOpts} from "../../../src/execution/engine/http.js"; import {IExecutionEngine, initializeExecutionEngine} from "../../../src/execution/index.js"; import { @@ -47,7 +48,7 @@ describe("ExecutionEngine / http", () => { retryAttempts: defaultExecutionEngineHttpOpts.retryAttempts, retryDelay: defaultExecutionEngineHttpOpts.retryDelay, }, - {signal: controller.signal} + {signal: controller.signal, logger: console as unknown as Logger} ); }); diff --git a/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts b/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts index 1b4735bbea88..2df5897dae7a 100644 --- a/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts @@ -3,6 +3,7 @@ import {fastify} from "fastify"; import {fromHexString} from "@chainsafe/ssz"; import {ForkName} from "@lodestar/params"; +import {Logger} from "@lodestar/logger"; import {defaultExecutionEngineHttpOpts} from "../../../src/execution/engine/http.js"; import {bytesToData, numToQuantity} from "../../../src/eth1/provider/utils.js"; import {IExecutionEngine, initializeExecutionEngine, PayloadAttributes} from "../../../src/execution/index.js"; @@ -52,7 +53,7 @@ describe("ExecutionEngine / http ", () => { retryAttempts: defaultExecutionEngineHttpOpts.retryAttempts, retryDelay: defaultExecutionEngineHttpOpts.retryDelay, }, - {signal: controller.signal} + {signal: controller.signal, logger: console as unknown as Logger} ); }); diff --git a/packages/cli/test/sim/endpoints.test.ts b/packages/cli/test/sim/endpoints.test.ts index 3fe294f2324b..955214863ed5 100644 --- a/packages/cli/test/sim/endpoints.test.ts +++ b/packages/cli/test/sim/endpoints.test.ts @@ -125,6 +125,7 @@ await env.tracker.assert("BN Not Synced", async () => { syncDistance: "0", isSyncing: false, isOptimistic: false, + elOffline: false, }; const res = await node.api.node.getSyncingStatus(); From 4b5280dbeedf6b1737986043ed571066974e0010 Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Sat, 15 Jul 2023 01:23:02 +0700 Subject: [PATCH 62/96] feat: prioritize api signature sets (#5754) * feat: enhance LinkedList with an iterable iterator * feat: use LinkedList for bls multithread impl * feat: enhance LinkedList with unshift() method * feat: enhance verifySignatureSets api with priority option * feat: prioritize signature sets from api * fix: AggregateAndProof perf test * fix: attestation.test.ts perf test * fix: add prioritizedJobs to bufferedJobs of bls multithread * chore: add metrics to track verifySignatureSets options --- .../src/api/impl/beacon/pool/index.ts | 24 +++--- .../src/api/impl/validator/index.ts | 9 +-- .../beacon-node/src/chain/bls/interface.ts | 4 + .../src/chain/bls/multithread/index.ts | 35 ++++++-- .../src/chain/validation/aggregateAndProof.ts | 27 ++++++- .../src/chain/validation/attestation.ts | 59 +++++++++++--- .../src/chain/validation/attesterSlashing.ts | 18 ++++- .../chain/validation/blsToExecutionChange.ts | 23 +++++- .../src/chain/validation/proposerSlashing.ts | 18 ++++- .../src/chain/validation/syncCommittee.ts | 16 +++- .../src/chain/validation/voluntaryExit.ts | 18 ++++- .../src/metrics/metrics/lodestar.ts | 12 +++ .../src/network/processor/gossipHandlers.ts | 6 +- packages/beacon-node/src/util/array.ts | 33 ++++++++ .../validation/aggregateAndProof.test.ts | 18 ++++- .../perf/chain/validation/attestation.test.ts | 17 +++- .../validation/aggregateAndProof.test.ts | 10 ++- .../unit/chain/validation/attestation.test.ts | 81 +++++++++---------- .../validation/blsToExecutionChange.test.ts | 14 ++-- .../beacon-node/test/unit/util/array.test.ts | 72 +++++++++++++++++ 20 files changed, 402 insertions(+), 112 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index 15bb1a3107db..f1973fef615e 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -1,12 +1,12 @@ import {routes, ServerApi} from "@lodestar/api"; import {Epoch, ssz} from "@lodestar/types"; import {SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params"; -import {validateGossipAttestation} from "../../../../chain/validation/index.js"; -import {validateGossipAttesterSlashing} from "../../../../chain/validation/attesterSlashing.js"; -import {validateGossipProposerSlashing} from "../../../../chain/validation/proposerSlashing.js"; -import {validateGossipVoluntaryExit} from "../../../../chain/validation/voluntaryExit.js"; -import {validateBlsToExecutionChange} from "../../../../chain/validation/blsToExecutionChange.js"; -import {validateSyncCommitteeSigOnly} from "../../../../chain/validation/syncCommittee.js"; +import {validateApiAttestation} from "../../../../chain/validation/index.js"; +import {validateApiAttesterSlashing} from "../../../../chain/validation/attesterSlashing.js"; +import {validateApiProposerSlashing} from "../../../../chain/validation/proposerSlashing.js"; +import {validateApiVoluntaryExit} from "../../../../chain/validation/voluntaryExit.js"; +import {validateApiBlsToExecutionChange} from "../../../../chain/validation/blsToExecutionChange.js"; +import {validateApiSyncCommittee} from "../../../../chain/validation/syncCommittee.js"; import {ApiModules} from "../../types.js"; import {AttestationError, GossipAction, SyncCommitteeError} from "../../../../chain/errors/index.js"; import {validateGossipFnRetryUnknownRoot} from "../../../../network/processor/gossipHandlers.js"; @@ -53,7 +53,7 @@ export function getBeaconPoolApi({ attestations.map(async (attestation, i) => { try { // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - const validateFn = () => validateGossipAttestation(chain, {attestation, serializedData: null}, null); + const validateFn = () => validateApiAttestation(chain, {attestation, serializedData: null}); const {slot, beaconBlockRoot} = attestation.data; // when a validator is configured with multiple beacon node urls, this attestation data may come from another beacon node // and the block hasn't been in our forkchoice since we haven't seen / processing that block @@ -97,19 +97,19 @@ export function getBeaconPoolApi({ }, async submitPoolAttesterSlashings(attesterSlashing) { - await validateGossipAttesterSlashing(chain, attesterSlashing); + await validateApiAttesterSlashing(chain, attesterSlashing); chain.opPool.insertAttesterSlashing(attesterSlashing); await network.publishAttesterSlashing(attesterSlashing); }, async submitPoolProposerSlashings(proposerSlashing) { - await validateGossipProposerSlashing(chain, proposerSlashing); + await validateApiProposerSlashing(chain, proposerSlashing); chain.opPool.insertProposerSlashing(proposerSlashing); await network.publishProposerSlashing(proposerSlashing); }, async submitPoolVoluntaryExit(voluntaryExit) { - await validateGossipVoluntaryExit(chain, voluntaryExit); + await validateApiVoluntaryExit(chain, voluntaryExit); chain.opPool.insertVoluntaryExit(voluntaryExit); chain.emitter.emit(routes.events.EventType.voluntaryExit, voluntaryExit); await network.publishVoluntaryExit(voluntaryExit); @@ -122,7 +122,7 @@ export function getBeaconPoolApi({ blsToExecutionChanges.map(async (blsToExecutionChange, i) => { try { // Ignore even if the change exists and reprocess - await validateBlsToExecutionChange(chain, blsToExecutionChange, true); + await validateApiBlsToExecutionChange(chain, blsToExecutionChange); const preCapella = chain.clock.currentEpoch < chain.config.CAPELLA_FORK_EPOCH; chain.opPool.insertBlsToExecutionChange(blsToExecutionChange, preCapella); @@ -182,7 +182,7 @@ export function getBeaconPoolApi({ // Verify signature only, all other data is very likely to be correct, since the `signature` object is created by this node. // Worst case if `signature` is not valid, gossip peers will drop it and slightly downscore us. - await validateSyncCommitteeSigOnly(chain, state, signature); + await validateApiSyncCommittee(chain, state, signature); // The same validator can appear multiple times in the sync committee. It can appear multiple times per // subnet even. First compute on which subnet the signature must be broadcasted to. diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 42e3783fa920..886fe9ab3ae4 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -20,7 +20,7 @@ import {Root, Slot, ValidatorIndex, ssz, Epoch, ProducedBlockSource, bellatrix, import {ExecutionStatus} from "@lodestar/fork-choice"; import {toHex} from "@lodestar/utils"; import {AttestationError, AttestationErrorCode, GossipAction, SyncCommitteeError} from "../../../chain/errors/index.js"; -import {validateGossipAggregateAndProof} from "../../../chain/validation/index.js"; +import {validateApiAggregateAndProof} from "../../../chain/validation/index.js"; import {ZERO_HASH} from "../../../constants/index.js"; import {SyncState} from "../../../sync/index.js"; import {isOptimisticBlock} from "../../../util/forkChoice.js"; @@ -554,12 +554,7 @@ export function getValidatorApi({ try { // TODO: Validate in batch // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - const validateFn = () => - validateGossipAggregateAndProof( - chain, - signedAggregateAndProof, - true // skip known attesters check - ); + const validateFn = () => validateApiAggregateAndProof(chain, signedAggregateAndProof); const {slot, beaconBlockRoot} = signedAggregateAndProof.message.aggregate.data; // when a validator is configured with multiple beacon node urls, this attestation may come from another beacon node // and the block hasn't been in our forkchoice since we haven't seen / processing that block diff --git a/packages/beacon-node/src/chain/bls/interface.ts b/packages/beacon-node/src/chain/bls/interface.ts index 2abeca62e103..0b704e91e201 100644 --- a/packages/beacon-node/src/chain/bls/interface.ts +++ b/packages/beacon-node/src/chain/bls/interface.ts @@ -15,6 +15,10 @@ export type VerifySignatureOpts = { * Ignore the batchable option if this is true. */ verifyOnMainThread?: boolean; + /** + * Some signature sets are more important than others, and should be verified first. + */ + priority?: boolean; }; export interface IBlsVerifier { diff --git a/packages/beacon-node/src/chain/bls/multithread/index.ts b/packages/beacon-node/src/chain/bls/multithread/index.ts index c350ef37bf2e..7c3141454ce2 100644 --- a/packages/beacon-node/src/chain/bls/multithread/index.ts +++ b/packages/beacon-node/src/chain/bls/multithread/index.ts @@ -15,6 +15,7 @@ import {Metrics} from "../../../metrics/index.js"; import {IBlsVerifier, VerifySignatureOpts} from "../interface.js"; import {getAggregatedPubkey, getAggregatedPubkeysCount} from "../utils.js"; import {verifySignatureSetsMaybeBatch} from "../maybeBatch.js"; +import {LinkedList} from "../../../util/array.js"; import {BlsWorkReq, BlsWorkResult, WorkerData, WorkResultCode} from "./types.js"; import {chunkifyMaximizeChunkSize} from "./utils.js"; import {defaultPoolSize} from "./poolSize.js"; @@ -106,9 +107,10 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { private readonly format: PointFormat; private readonly workers: WorkerDescriptor[]; - private readonly jobs: JobQueueItem[] = []; + private readonly jobs = new LinkedList(); private bufferedJobs: { - jobs: JobQueueItem[]; + jobs: LinkedList; + prioritizedJobs: LinkedList; sigCount: number; firstPush: number; timeout: NodeJS.Timeout; @@ -151,6 +153,13 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { async verifySignatureSets(sets: ISignatureSet[], opts: VerifySignatureOpts = {}): Promise { // Pubkeys are aggregated in the main thread regardless if verified in workers or in main thread this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets)); + this.metrics?.blsThreadPool.totalSigSets.inc(sets.length); + if (opts.priority) { + this.metrics?.blsThreadPool.prioritizedSigSets.inc(sets.length); + } + if (opts.batchable) { + this.metrics?.blsThreadPool.batchableSigSets.inc(sets.length); + } if (opts.verifyOnMainThread && !this.blsVerifyAllMultiThread) { const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer(); @@ -199,7 +208,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { for (const job of this.jobs) { job.reject(new QueueError({code: QueueErrorCode.QUEUE_ABORTED})); } - this.jobs.splice(0, this.jobs.length); + this.jobs.clear(); // Terminate all workers. await to ensure no workers are left hanging await Promise.all( @@ -272,18 +281,21 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { return new Promise((resolve, reject) => { const job = {resolve, reject, addedTimeMs: Date.now(), workReq}; + // below is for non-priority jobs // Append batchable sets to `bufferedJobs`, starting a timeout to push them into `jobs`. // Do not call `runJob()`, it is called from `runBufferedJobs()` if (workReq.opts.batchable) { if (!this.bufferedJobs) { this.bufferedJobs = { - jobs: [], + jobs: new LinkedList(), + prioritizedJobs: new LinkedList(), sigCount: 0, firstPush: Date.now(), timeout: setTimeout(this.runBufferedJobs, MAX_BUFFER_WAIT_MS), }; } - this.bufferedJobs.jobs.push(job); + const jobs = workReq.opts.priority ? this.bufferedJobs.prioritizedJobs : this.bufferedJobs.jobs; + jobs.push(job); this.bufferedJobs.sigCount += job.workReq.sets.length; if (this.bufferedJobs.sigCount > MAX_BUFFERED_SIGS) { clearTimeout(this.bufferedJobs.timeout); @@ -295,7 +307,11 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { // This is useful to allow batching job submitted from a synchronous for loop, // and to prevent large stacks since runJob may be called recursively. else { - this.jobs.push(job); + if (workReq.opts.priority) { + this.jobs.unshift(job); + } else { + this.jobs.push(job); + } setTimeout(this.runJob, 0); } }); @@ -424,7 +440,12 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { */ private runBufferedJobs = (): void => { if (this.bufferedJobs) { - this.jobs.push(...this.bufferedJobs.jobs); + for (const job of this.bufferedJobs.jobs) { + this.jobs.push(job); + } + for (const job of this.bufferedJobs.prioritizedJobs) { + this.jobs.unshift(job); + } this.bufferedJobs = null; setTimeout(this.runJob, 0); } diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 982818a48a7a..aa60140e2d9a 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -25,12 +25,33 @@ export type AggregateAndProofValidationResult = { attDataRootHex: RootHex; }; +export async function validateApiAggregateAndProof( + chain: IBeaconChain, + signedAggregateAndProof: phase0.SignedAggregateAndProof +): Promise { + const skipValidationKnownAttesters = true; + const prioritizeBls = true; + return validateAggregateAndProof(chain, signedAggregateAndProof, null, {skipValidationKnownAttesters, prioritizeBls}); +} + export async function validateGossipAggregateAndProof( chain: IBeaconChain, signedAggregateAndProof: phase0.SignedAggregateAndProof, - skipValidationKnownAttesters = false, - serializedData: Uint8Array | null = null + serializedData: Uint8Array +): Promise { + return validateAggregateAndProof(chain, signedAggregateAndProof, serializedData); +} + +async function validateAggregateAndProof( + chain: IBeaconChain, + signedAggregateAndProof: phase0.SignedAggregateAndProof, + serializedData: Uint8Array | null = null, + opts: {skipValidationKnownAttesters: boolean; prioritizeBls: boolean} = { + skipValidationKnownAttesters: false, + prioritizeBls: false, + } ): Promise { + const {skipValidationKnownAttesters, prioritizeBls} = opts; // Do checks in this order: // - do early checks (w/o indexed attestation) // - > obtain indexed attestation and committes per slot @@ -176,7 +197,7 @@ export async function validateGossipAggregateAndProof( ]; // no need to write to SeenAttestationDatas - if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true}))) { + if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) { throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.INVALID_SIGNATURE}); } diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index dc3a963d3bdc..3a17b3d4a0aa 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -29,16 +29,18 @@ export type AttestationValidationResult = { attDataRootHex: RootHex; }; -export type AttestationOrBytes = - // for api - | {attestation: phase0.Attestation; serializedData: null} - // for gossip - | { - attestation: null; - serializedData: Uint8Array; - // available in NetworkProcessor since we check for unknown block root attestations - attSlot: Slot; - }; +export type AttestationOrBytes = ApiAttestation | GossipAttestation; + +/** attestation from api */ +export type ApiAttestation = {attestation: phase0.Attestation; serializedData: null}; + +/** attestation from gossip */ +export type GossipAttestation = { + attestation: null; + serializedData: Uint8Array; + // available in NetworkProcessor since we check for unknown block root attestations + attSlot: Slot; +}; /** * The beacon chain shufflings are designed to provide 1 epoch lookahead @@ -46,15 +48,46 @@ export type AttestationOrBytes = */ const SHUFFLING_LOOK_AHEAD_EPOCHS = 1; +/** + * Validate attestations from gossip + * - Only deserialize the attestation if needed, use the cached AttestationData instead + * - This is to avoid deserializing similar attestation multiple times which could help the gc + * - subnet is required + * - do not prioritize bls signature set + */ +export async function validateGossipAttestation( + chain: IBeaconChain, + attestationOrBytes: GossipAttestation, + /** Optional, to allow verifying attestations through API with unknown subnet */ + subnet: number +): Promise { + return validateAttestation(chain, attestationOrBytes, subnet); +} + +/** + * Validate attestations from api + * - no need to deserialize attestation + * - no subnet + * - prioritize bls signature set + */ +export async function validateApiAttestation( + chain: IBeaconChain, + attestationOrBytes: ApiAttestation +): Promise { + const prioritizeBls = true; + return validateAttestation(chain, attestationOrBytes, null, prioritizeBls); +} + /** * Only deserialize the attestation if needed, use the cached AttestationData instead * This is to avoid deserializing similar attestation multiple times which could help the gc */ -export async function validateGossipAttestation( +async function validateAttestation( chain: IBeaconChain, attestationOrBytes: AttestationOrBytes, /** Optional, to allow verifying attestations through API with unknown subnet */ - subnet: number | null + subnet: number | null, + prioritizeBls = false ): Promise { // Do checks in this order: // - do early checks (w/o indexed attestation) @@ -268,7 +301,7 @@ export async function validateGossipAttestation( } } - if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true}))) { + if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true, priority: prioritizeBls}))) { throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.INVALID_SIGNATURE}); } diff --git a/packages/beacon-node/src/chain/validation/attesterSlashing.ts b/packages/beacon-node/src/chain/validation/attesterSlashing.ts index b1f800264f82..818812526fb3 100644 --- a/packages/beacon-node/src/chain/validation/attesterSlashing.ts +++ b/packages/beacon-node/src/chain/validation/attesterSlashing.ts @@ -7,9 +7,25 @@ import { import {IBeaconChain} from ".."; import {AttesterSlashingError, AttesterSlashingErrorCode, GossipAction} from "../errors/index.js"; +export async function validateApiAttesterSlashing( + chain: IBeaconChain, + attesterSlashing: phase0.AttesterSlashing +): Promise { + const prioritizeBls = true; + return validateAttesterSlashing(chain, attesterSlashing, prioritizeBls); +} + export async function validateGossipAttesterSlashing( chain: IBeaconChain, attesterSlashing: phase0.AttesterSlashing +): Promise { + return validateAttesterSlashing(chain, attesterSlashing); +} + +export async function validateAttesterSlashing( + chain: IBeaconChain, + attesterSlashing: phase0.AttesterSlashing, + prioritizeBls = false ): Promise { // [IGNORE] At least one index in the intersection of the attesting indices of each attestation has not yet been seen // in any prior attester_slashing (i.e. @@ -36,7 +52,7 @@ export async function validateGossipAttesterSlashing( } const signatureSets = getAttesterSlashingSignatureSets(state, attesterSlashing); - if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true}))) { + if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) { throw new AttesterSlashingError(GossipAction.REJECT, { code: AttesterSlashingErrorCode.INVALID, error: Error("Invalid signature"), diff --git a/packages/beacon-node/src/chain/validation/blsToExecutionChange.ts b/packages/beacon-node/src/chain/validation/blsToExecutionChange.ts index c2c183aa35a3..e5ad56daeb5c 100644 --- a/packages/beacon-node/src/chain/validation/blsToExecutionChange.ts +++ b/packages/beacon-node/src/chain/validation/blsToExecutionChange.ts @@ -7,11 +7,28 @@ import { import {IBeaconChain} from ".."; import {BlsToExecutionChangeError, BlsToExecutionChangeErrorCode, GossipAction} from "../errors/index.js"; -export async function validateBlsToExecutionChange( +export async function validateApiBlsToExecutionChange( + chain: IBeaconChain, + blsToExecutionChange: capella.SignedBLSToExecutionChange +): Promise { + const ignoreExists = true; + const prioritizeBls = true; + return validateBlsToExecutionChange(chain, blsToExecutionChange, {ignoreExists, prioritizeBls}); +} + +export async function validateGossipBlsToExecutionChange( + chain: IBeaconChain, + blsToExecutionChange: capella.SignedBLSToExecutionChange +): Promise { + return validateBlsToExecutionChange(chain, blsToExecutionChange); +} + +async function validateBlsToExecutionChange( chain: IBeaconChain, blsToExecutionChange: capella.SignedBLSToExecutionChange, - ignoreExists = false + opts: {ignoreExists?: boolean; prioritizeBls?: boolean} = {ignoreExists: false, prioritizeBls: false} ): Promise { + const {ignoreExists, prioritizeBls} = opts; // [IGNORE] The blsToExecutionChange is the first valid blsToExecutionChange received for the validator with index // signedBLSToExecutionChange.message.validatorIndex. if (!ignoreExists && chain.opPool.hasSeenBlsToExecutionChange(blsToExecutionChange.message.validatorIndex)) { @@ -36,7 +53,7 @@ export async function validateBlsToExecutionChange( } const signatureSet = getBlsToExecutionChangeSignatureSet(config, blsToExecutionChange); - if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true}))) { + if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true, priority: prioritizeBls}))) { throw new BlsToExecutionChangeError(GossipAction.REJECT, { code: BlsToExecutionChangeErrorCode.INVALID_SIGNATURE, }); diff --git a/packages/beacon-node/src/chain/validation/proposerSlashing.ts b/packages/beacon-node/src/chain/validation/proposerSlashing.ts index 543f4ded892b..7281302aae8d 100644 --- a/packages/beacon-node/src/chain/validation/proposerSlashing.ts +++ b/packages/beacon-node/src/chain/validation/proposerSlashing.ts @@ -3,9 +3,25 @@ import {assertValidProposerSlashing, getProposerSlashingSignatureSets} from "@lo import {IBeaconChain} from ".."; import {ProposerSlashingError, ProposerSlashingErrorCode, GossipAction} from "../errors/index.js"; +export async function validateApiProposerSlashing( + chain: IBeaconChain, + proposerSlashing: phase0.ProposerSlashing +): Promise { + const prioritizeBls = true; + return validateProposerSlashing(chain, proposerSlashing, prioritizeBls); +} + export async function validateGossipProposerSlashing( chain: IBeaconChain, proposerSlashing: phase0.ProposerSlashing +): Promise { + return validateProposerSlashing(chain, proposerSlashing); +} + +async function validateProposerSlashing( + chain: IBeaconChain, + proposerSlashing: phase0.ProposerSlashing, + prioritizeBls = false ): Promise { // [IGNORE] The proposer slashing is the first valid proposer slashing received for the proposer with index // proposer_slashing.signed_header_1.message.proposer_index. @@ -29,7 +45,7 @@ export async function validateGossipProposerSlashing( } const signatureSets = getProposerSlashingSignatureSets(state, proposerSlashing); - if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true}))) { + if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) { throw new ProposerSlashingError(GossipAction.REJECT, { code: ProposerSlashingErrorCode.INVALID, error: Error("Invalid signature"), diff --git a/packages/beacon-node/src/chain/validation/syncCommittee.ts b/packages/beacon-node/src/chain/validation/syncCommittee.ts index f5450d0b26ec..43a4c95c59da 100644 --- a/packages/beacon-node/src/chain/validation/syncCommittee.ts +++ b/packages/beacon-node/src/chain/validation/syncCommittee.ts @@ -71,16 +71,26 @@ export async function validateGossipSyncCommittee( return {indexInSubcommittee}; } +export async function validateApiSyncCommittee( + chain: IBeaconChain, + headState: CachedBeaconStateAllForks, + syncCommittee: altair.SyncCommitteeMessage +): Promise { + const prioritizeBls = true; + return validateSyncCommitteeSigOnly(chain, headState, syncCommittee, prioritizeBls); +} + /** * Abstracted so it can be re-used in API validation. */ -export async function validateSyncCommitteeSigOnly( +async function validateSyncCommitteeSigOnly( chain: IBeaconChain, headState: CachedBeaconStateAllForks, - syncCommittee: altair.SyncCommitteeMessage + syncCommittee: altair.SyncCommitteeMessage, + prioritizeBls = false ): Promise { const signatureSet = getSyncCommitteeSignatureSet(headState, syncCommittee); - if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true}))) { + if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true, priority: prioritizeBls}))) { throw new SyncCommitteeError(GossipAction.REJECT, { code: SyncCommitteeErrorCode.INVALID_SIGNATURE, }); diff --git a/packages/beacon-node/src/chain/validation/voluntaryExit.ts b/packages/beacon-node/src/chain/validation/voluntaryExit.ts index 385c370b8de2..3957b51180d9 100644 --- a/packages/beacon-node/src/chain/validation/voluntaryExit.ts +++ b/packages/beacon-node/src/chain/validation/voluntaryExit.ts @@ -4,9 +4,25 @@ import {IBeaconChain} from ".."; import {VoluntaryExitError, VoluntaryExitErrorCode, GossipAction} from "../errors/index.js"; import {RegenCaller} from "../regen/index.js"; +export async function validateApiVoluntaryExit( + chain: IBeaconChain, + voluntaryExit: phase0.SignedVoluntaryExit +): Promise { + const prioritizeBls = true; + return validateVoluntaryExit(chain, voluntaryExit, prioritizeBls); +} + export async function validateGossipVoluntaryExit( chain: IBeaconChain, voluntaryExit: phase0.SignedVoluntaryExit +): Promise { + return validateVoluntaryExit(chain, voluntaryExit); +} + +async function validateVoluntaryExit( + chain: IBeaconChain, + voluntaryExit: phase0.SignedVoluntaryExit, + prioritizeBls = false ): Promise { // [IGNORE] The voluntary exit is the first valid voluntary exit received for the validator with index // signed_voluntary_exit.message.validator_index. @@ -34,7 +50,7 @@ export async function validateGossipVoluntaryExit( } const signatureSet = getVoluntaryExitSignatureSet(state, voluntaryExit); - if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true}))) { + if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true, priority: prioritizeBls}))) { throw new VoluntaryExitError(GossipAction.REJECT, { code: VoluntaryExitErrorCode.INVALID_SIGNATURE, }); diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 082f3680e161..8beaacc165a3 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -374,6 +374,18 @@ export function createLodestarMetrics( // Time per sig ~0.9ms on good machines buckets: [0.5e-3, 0.75e-3, 1e-3, 1.5e-3, 2e-3, 5e-3], }), + totalSigSets: register.gauge({ + name: "lodestar_bls_thread_pool_sig_sets_total", + help: "Count of total signature sets", + }), + prioritizedSigSets: register.gauge({ + name: "lodestar_bls_thread_pool_prioritized_sig_sets_total", + help: "Count of total prioritized signature sets", + }), + batchableSigSets: register.gauge({ + name: "lodestar_bls_thread_pool_batchable_sig_sets_total", + help: "Count of total batchable signature sets", + }), }, // BLS time on single thread mode diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 455a4d34c1dc..e3692b7fb4a3 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -27,7 +27,7 @@ import { validateGossipSyncCommittee, validateSyncCommitteeGossipContributionAndProof, validateGossipVoluntaryExit, - validateBlsToExecutionChange, + validateGossipBlsToExecutionChange, AttestationValidationResult, AggregateAndProofValidationResult, } from "../../chain/validation/index.js"; @@ -194,7 +194,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH const signedAggregateAndProof = sszDeserialize(topic, serializedData); try { - validationResult = await validateGossipAggregateAndProof(chain, signedAggregateAndProof, false, serializedData); + validationResult = await validateGossipAggregateAndProof(chain, signedAggregateAndProof, serializedData); } catch (e) { if (e instanceof AttestationError && e.action === GossipAction.REJECT) { chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "gossip_reject"); @@ -376,7 +376,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH // blsToExecutionChange is to be generated and validated against GENESIS_FORK_VERSION [GossipType.bls_to_execution_change]: async ({serializedData}, topic) => { const blsToExecutionChange = sszDeserialize(topic, serializedData); - await validateBlsToExecutionChange(chain, blsToExecutionChange); + await validateGossipBlsToExecutionChange(chain, blsToExecutionChange); // Handler try { diff --git a/packages/beacon-node/src/util/array.ts b/packages/beacon-node/src/util/array.ts index f93d323fd5ac..a0ed94c35450 100644 --- a/packages/beacon-node/src/util/array.ts +++ b/packages/beacon-node/src/util/array.ts @@ -63,6 +63,25 @@ export class LinkedList { this._length++; } + unshift(data: T): void { + if (this._length === 0) { + this.tail = this.head = new Node(data); + this._length++; + return; + } + + if (!this.head || !this.tail) { + // should not happen + throw Error("No head or tail"); + } + + const newHead = new Node(data); + newHead.next = this.head; + this.head.prev = newHead; + this.head = newHead; + this._length++; + } + pop(): T | null { const oldTail = this.tail; if (!oldTail) return null; @@ -100,6 +119,20 @@ export class LinkedList { this._length = 0; } + [Symbol.iterator](): Iterator { + let node = this.head; + return { + next(): IteratorResult { + if (!node) { + return {done: true, value: undefined}; + } + const value = node.data; + node = node.next; + return {done: false, value}; + }, + }; + } + toArray(): T[] { let node = this.head; if (!node) return []; diff --git a/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts index 6b0e9acc6c68..fe8ade59cef2 100644 --- a/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts @@ -1,7 +1,8 @@ import {itBench} from "@dapplion/benchmark"; +import {ssz} from "@lodestar/types"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; -import {validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; +import {validateApiAggregateAndProof, validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; import {getAggregateAndProofValidData} from "../../../utils/validationData/aggregateAndProof.js"; describe("validate gossip signedAggregateAndProof", () => { @@ -16,6 +17,19 @@ describe("validate gossip signedAggregateAndProof", () => { const aggStruct = signedAggregateAndProof; for (const [id, agg] of Object.entries({struct: aggStruct})) { + const serializedData = ssz.phase0.SignedAggregateAndProof.serialize(aggStruct); + + itBench({ + id: `validate api signedAggregateAndProof - ${id}`, + beforeEach: () => { + chain.seenAggregators["validatorIndexesByEpoch"].clear(); + chain.seenAggregatedAttestations["aggregateRootsByEpoch"].clear(); + }, + fn: async () => { + await validateApiAggregateAndProof(chain, agg); + }, + }); + itBench({ id: `validate gossip signedAggregateAndProof - ${id}`, beforeEach: () => { @@ -23,7 +37,7 @@ describe("validate gossip signedAggregateAndProof", () => { chain.seenAggregatedAttestations["aggregateRootsByEpoch"].clear(); }, fn: async () => { - await validateGossipAggregateAndProof(chain, agg); + await validateGossipAggregateAndProof(chain, agg, serializedData); }, }); } diff --git a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts index ed289444f8d0..b9664c6f5efd 100644 --- a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts @@ -1,10 +1,11 @@ import {itBench} from "@dapplion/benchmark"; +import {ssz} from "@lodestar/types"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; -import {validateGossipAttestation} from "../../../../src/chain/validation/index.js"; +import {validateApiAttestation, validateGossipAttestation} from "../../../../src/chain/validation/index.js"; import {getAttestationValidData} from "../../../utils/validationData/attestation.js"; -describe("validate gossip attestation", () => { +describe("validate attestation", () => { const vc = 64; const stateSlot = 100; @@ -16,11 +17,21 @@ describe("validate gossip attestation", () => { const attStruct = attestation; for (const [id, att] of Object.entries({struct: attStruct})) { + const serializedData = ssz.phase0.Attestation.serialize(att); + const slot = attestation.data.slot; + itBench({ + id: `validate api attestation - ${id}`, + beforeEach: () => chain.seenAttesters["validatorIndexesByEpoch"].clear(), + fn: async () => { + await validateApiAttestation(chain, {attestation: att, serializedData: null}); + }, + }); + itBench({ id: `validate gossip attestation - ${id}`, beforeEach: () => chain.seenAttesters["validatorIndexesByEpoch"].clear(), fn: async () => { - await validateGossipAttestation(chain, {attestation: att, serializedData: null}, subnet); + await validateGossipAttestation(chain, {attestation: null, serializedData, attSlot: slot}, subnet); }, }); } diff --git a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts index f3b934ac2242..54191ba34846 100644 --- a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts @@ -6,7 +6,7 @@ import {processSlots} from "@lodestar/state-transition"; import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {IBeaconChain} from "../../../../src/chain/index.js"; import {AttestationErrorCode} from "../../../../src/chain/errors/index.js"; -import {validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; +import {validateApiAggregateAndProof, validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; import {expectRejectedWithLodestarError} from "../../../utils/errors.js"; import {memoOnce} from "../../../utils/cache.js"; import { @@ -42,7 +42,7 @@ describe("chain / validation / aggregateAndProof", () => { it("Valid", async () => { const {chain, signedAggregateAndProof} = getValidData({}); - await validateGossipAggregateAndProof(chain, signedAggregateAndProof); + await validateApiAggregateAndProof(chain, signedAggregateAndProof); }); it("BAD_TARGET_EPOCH", async () => { @@ -188,6 +188,10 @@ describe("chain / validation / aggregateAndProof", () => { signedAggregateAndProof: phase0.SignedAggregateAndProof, errorCode: AttestationErrorCode ): Promise { - await expectRejectedWithLodestarError(validateGossipAggregateAndProof(chain, signedAggregateAndProof), errorCode); + const serializedData = ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof); + await expectRejectedWithLodestarError( + validateGossipAggregateAndProof(chain, signedAggregateAndProof, serializedData), + errorCode + ); } }); diff --git a/packages/beacon-node/test/unit/chain/validation/attestation.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation.test.ts index d56cb261a8a5..6296ecb4c6fc 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation.test.ts @@ -11,8 +11,10 @@ import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state- import {IBeaconChain} from "../../../../src/chain/index.js"; import {AttestationErrorCode, GossipErrorCode} from "../../../../src/chain/errors/index.js"; import { - AttestationOrBytes, + ApiAttestation, + GossipAttestation, getStateForAttestationVerification, + validateApiAttestation, validateGossipAttestation, } from "../../../../src/chain/validation/index.js"; import {expectRejectedWithLodestarError} from "../../../utils/errors.js"; @@ -48,14 +50,14 @@ describe("chain / validation / attestation", () => { } it("Valid", async () => { - const {chain, attestation, subnet} = getValidData(); + const {chain, attestation} = getValidData(); - await validateGossipAttestation(chain, {attestation, serializedData: null}, subnet); + await validateApiAttestation(chain, {attestation, serializedData: null}); }); it("INVALID_SERIALIZED_BYTES_ERROR_CODE", async () => { const {chain, subnet} = getValidData(); - await expectError( + await expectGossipError( chain, {attestation: null, serializedData: Buffer.alloc(0), attSlot: 0}, subnet, @@ -70,8 +72,8 @@ describe("chain / validation / attestation", () => { attestation.data.target.epoch += 1; const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError(chain, {attestation, serializedData: null}, subnet, AttestationErrorCode.BAD_TARGET_EPOCH); - await expectError( + await expectApiError(chain, {attestation, serializedData: null}, AttestationErrorCode.BAD_TARGET_EPOCH); + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -84,8 +86,8 @@ describe("chain / validation / attestation", () => { const {chain, attestation, subnet} = getValidData({attSlot: stateSlot - SLOTS_PER_EPOCH - 3}); const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError(chain, {attestation, serializedData: null}, subnet, AttestationErrorCode.PAST_SLOT); - await expectError( + await expectApiError(chain, {attestation, serializedData: null}, AttestationErrorCode.PAST_SLOT); + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -98,8 +100,8 @@ describe("chain / validation / attestation", () => { const {chain, attestation, subnet} = getValidData({attSlot: stateSlot + 2}); const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError(chain, {attestation, serializedData: null}, subnet, AttestationErrorCode.FUTURE_SLOT); - await expectError( + await expectApiError(chain, {attestation, serializedData: null}, AttestationErrorCode.FUTURE_SLOT); + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -114,13 +116,12 @@ describe("chain / validation / attestation", () => { attestation.aggregationBits.set(bitIndex, false); const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError( + await expectApiError( chain, {attestation, serializedData: null}, - subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET ); - await expectError( + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -135,7 +136,7 @@ describe("chain / validation / attestation", () => { attestation.aggregationBits.set(bitIndex + 1, true); const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError( + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -149,13 +150,12 @@ describe("chain / validation / attestation", () => { attestation.data.beaconBlockRoot = UNKNOWN_ROOT; const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError( + await expectApiError( chain, {attestation, serializedData: null}, - subnet, AttestationErrorCode.UNKNOWN_OR_PREFINALIZED_BEACON_BLOCK_ROOT ); - await expectError( + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -169,8 +169,8 @@ describe("chain / validation / attestation", () => { attestation.data.target.root = UNKNOWN_ROOT; const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError(chain, {attestation, serializedData: null}, subnet, AttestationErrorCode.INVALID_TARGET_ROOT); - await expectError( + await expectApiError(chain, {attestation, serializedData: null}, AttestationErrorCode.INVALID_TARGET_ROOT); + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -189,13 +189,12 @@ describe("chain / validation / attestation", () => { } as Partial as IStateRegenerator; const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError( + await expectApiError( chain, {attestation, serializedData: null}, - subnet, AttestationErrorCode.NO_COMMITTEE_FOR_SLOT_AND_INDEX ); - await expectError( + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -212,13 +211,12 @@ describe("chain / validation / attestation", () => { ); const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError( + await expectApiError( chain, {attestation, serializedData: null}, - subnet, AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS ); - await expectError( + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -232,13 +230,7 @@ describe("chain / validation / attestation", () => { const invalidSubnet = subnet === 0 ? 1 : 0; const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError( - chain, - {attestation, serializedData: null}, - invalidSubnet, - AttestationErrorCode.INVALID_SUBNET_ID - ); - await expectError( + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, invalidSubnet, @@ -252,13 +244,8 @@ describe("chain / validation / attestation", () => { chain.seenAttesters.add(attestation.data.target.epoch, validatorIndex); const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError( - chain, - {attestation, serializedData: null}, - subnet, - AttestationErrorCode.ATTESTATION_ALREADY_KNOWN - ); - await expectError( + await expectApiError(chain, {attestation, serializedData: null}, AttestationErrorCode.ATTESTATION_ALREADY_KNOWN); + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -274,8 +261,8 @@ describe("chain / validation / attestation", () => { attestation.aggregationBits.set(bitIndex + 1, true); const serializedData = ssz.phase0.Attestation.serialize(attestation); - await expectError(chain, {attestation, serializedData: null}, subnet, AttestationErrorCode.INVALID_SIGNATURE); - await expectError( + await expectApiError(chain, {attestation, serializedData: null}, AttestationErrorCode.INVALID_SIGNATURE); + await expectGossipError( chain, {attestation: null, serializedData, attSlot: attestation.data.slot}, subnet, @@ -284,9 +271,17 @@ describe("chain / validation / attestation", () => { }); /** Alias to reduce code duplication */ - async function expectError( + async function expectApiError( + chain: IBeaconChain, + attestationOrBytes: ApiAttestation, + errorCode: string + ): Promise { + await expectRejectedWithLodestarError(validateApiAttestation(chain, attestationOrBytes), errorCode); + } + + async function expectGossipError( chain: IBeaconChain, - attestationOrBytes: AttestationOrBytes, + attestationOrBytes: GossipAttestation, subnet: number, errorCode: string ): Promise { diff --git a/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts b/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts index e3f3c0295655..8a100d315112 100644 --- a/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts @@ -19,7 +19,7 @@ import {createBeaconConfig} from "@lodestar/config"; import {BeaconChain} from "../../../../src/chain/index.js"; import {StubbedChainMutable} from "../../../utils/stub/index.js"; import {generateState} from "../../../utils/state.js"; -import {validateBlsToExecutionChange} from "../../../../src/chain/validation/blsToExecutionChange.js"; +import {validateGossipBlsToExecutionChange} from "../../../../src/chain/validation/blsToExecutionChange.js"; import {BlsToExecutionChangeErrorCode} from "../../../../src/chain/errors/blsToExecutionChangeError.js"; import {OpPool} from "../../../../src/chain/opPools/index.js"; import {expectRejectedWithLodestarError} from "../../../utils/errors.js"; @@ -115,13 +115,13 @@ describe("validate bls to execution change", () => { opPool.hasSeenBlsToExecutionChange.returns(true); await expectRejectedWithLodestarError( - validateBlsToExecutionChange(chainStub, signedBlsToExecChangeInvalid), + validateGossipBlsToExecutionChange(chainStub, signedBlsToExecChangeInvalid), BlsToExecutionChangeErrorCode.ALREADY_EXISTS ); }); it("should return valid blsToExecutionChange ", async () => { - await validateBlsToExecutionChange(chainStub, signedBlsToExecChange); + await validateGossipBlsToExecutionChange(chainStub, signedBlsToExecChange); }); it("should return invalid bls to execution Change - invalid validatorIndex", async () => { @@ -135,7 +135,7 @@ describe("validate bls to execution change", () => { }; await expectRejectedWithLodestarError( - validateBlsToExecutionChange(chainStub, signedBlsToExecChangeInvalid), + validateGossipBlsToExecutionChange(chainStub, signedBlsToExecChangeInvalid), BlsToExecutionChangeErrorCode.INVALID ); }); @@ -150,7 +150,7 @@ describe("validate bls to execution change", () => { }; await expectRejectedWithLodestarError( - validateBlsToExecutionChange(chainStub, signedBlsToExecChangeInvalid), + validateGossipBlsToExecutionChange(chainStub, signedBlsToExecChangeInvalid), BlsToExecutionChangeErrorCode.INVALID ); }); @@ -166,7 +166,7 @@ describe("validate bls to execution change", () => { }; await expectRejectedWithLodestarError( - validateBlsToExecutionChange(chainStub, signedBlsToExecChangeInvalid), + validateGossipBlsToExecutionChange(chainStub, signedBlsToExecChangeInvalid), BlsToExecutionChangeErrorCode.INVALID ); }); @@ -182,7 +182,7 @@ describe("validate bls to execution change", () => { }; await expectRejectedWithLodestarError( - validateBlsToExecutionChange(chainStub, signedBlsToExecChangeInvalid), + validateGossipBlsToExecutionChange(chainStub, signedBlsToExecChangeInvalid), BlsToExecutionChangeErrorCode.INVALID ); }); diff --git a/packages/beacon-node/test/unit/util/array.test.ts b/packages/beacon-node/test/unit/util/array.test.ts index e1556969bf19..88a970502243 100644 --- a/packages/beacon-node/test/unit/util/array.test.ts +++ b/packages/beacon-node/test/unit/util/array.test.ts @@ -53,6 +53,57 @@ describe("LinkedList", () => { expect(list.length).to.be.equal(0); }); + describe("push", () => { + const count = 100; + beforeEach(() => { + list = new LinkedList(); + expect(list.length).to.be.equal(0); + for (let i = 0; i < count; i++) list.push(i); + expect(list.length).to.be.equal(count); + expect(list.toArray()).to.be.deep.equal(Array.from({length: count}, (_, i) => i)); + }); + + it("push then pop", () => { + for (let i = 0; i < count; i++) { + expect(list.pop()).to.be.equal(count - i - 1); + } + expect(list.length).to.be.equal(0); + }); + + it("push then shift", () => { + for (let i = 0; i < count; i++) { + expect(list.shift()).to.be.equal(i); + } + expect(list.length).to.be.equal(0); + }); + }); + + describe("unshift", () => { + const count = 100; + beforeEach(() => { + list = new LinkedList(); + expect(list.length).to.be.equal(0); + for (let i = 0; i < count; i++) list.unshift(i); + expect(list.length).to.be.equal(count); + expect(list.toArray()).to.be.deep.equal(Array.from({length: count}, (_, i) => count - i - 1)); + }); + + it("unshift then pop", () => { + for (let i = 0; i < count; i++) { + expect(list.pop()).to.be.equal(i); + } + expect(list.length).to.be.equal(0); + }); + + it("unshift then shift", () => { + for (let i = 0; i < count; i++) { + expect(list.shift()).to.be.equal(count - i - 1); + } + + expect(list.length).to.be.equal(0); + }); + }); + it("toArray", () => { expect(list.toArray()).to.be.deep.equal([]); @@ -72,4 +123,25 @@ describe("LinkedList", () => { expect(list.toArray()).to.be.deep.equal([]); expect(list.length).to.be.equal(0); }); + + describe("iterator", () => { + const testCases: {count: number}[] = [{count: 0}, {count: 10}, {count: 100}]; + + for (const {count} of testCases) { + it(`should iterate over ${count} items`, () => { + for (let i = 0; i < count; i++) { + list.push(i); + } + + let i = 0; + for (const item of list) { + expect(item).to.be.equal(i); + i++; + } + + // make sure the list is the same + expect(list.toArray()).to.be.deep.equal(Array.from({length: count}, (_, i) => i)); + }); + } + }); }); From de00b80f16e59ad60bda535e0715b936e7df2481 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 14 Jul 2023 20:23:33 +0200 Subject: [PATCH 63/96] feat: track participants of produced aggregates (#5750) * Track participants of produced aggregates * Add benchmark * Fix typo in participants --------- Co-authored-by: Nico Flaig --- .../src/api/impl/validator/index.ts | 8 +++++++ .../chain/produceBlock/produceBlockBody.ts | 7 +++--- .../src/metrics/metrics/lodestar.ts | 24 +++++++++++++++++++ .../test/perf/util/bitArray.test.ts | 13 ++++++++++ 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 886fe9ab3ae4..242b6cccce9e 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -374,6 +374,11 @@ export function getValidatorApi({ const contribution = chain.syncCommitteeMessagePool.getContribution(subcommitteeIndex, slot, beaconBlockRoot); if (!contribution) throw new ApiError(500, "No contribution available"); + + metrics?.production.producedSyncContributionParticipants.observe( + contribution.aggregationBits.getTrueBitIndexes().length + ); + return {data: contribution}; }, @@ -538,6 +543,9 @@ export function getValidatorApi({ await waitForSlot(slot); // Must never request for a future slot > currentSlot + const aggregate = chain.attestationPool.getAggregate(slot, attestationDataRoot); + metrics?.production.producedAggregateParticipants.observe(aggregate.aggregationBits.getTrueBitIndexes().length); + return { data: chain.attestationPool.getAggregate(slot, attestationDataRoot), }; diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index e8a76ffc64b1..04b5760a4a71 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -130,10 +130,11 @@ export async function produceBlockBody( const blockEpoch = computeEpochAtSlot(blockSlot); if (blockEpoch >= this.config.ALTAIR_FORK_EPOCH) { - (blockBody as altair.BeaconBlockBody).syncAggregate = this.syncContributionAndProofPool.getAggregate( - parentSlot, - parentBlockRoot + const syncAggregate = this.syncContributionAndProofPool.getAggregate(parentSlot, parentBlockRoot); + this.metrics?.production.producedSyncAggregateParticipants.observe( + syncAggregate.syncCommitteeBits.getTrueBitIndexes().length ); + (blockBody as altair.BeaconBlockBody).syncAggregate = syncAggregate; } const fork = currentState.config.getForkName(blockSlot); diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 8beaacc165a3..f3a67700c53e 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -221,6 +221,30 @@ export function createLodestarMetrics( }), }, + production: { + producedAggregateParticipants: register.histogram({ + name: "lodestar_produced_aggregate_participants", + help: "API impl produced aggregates histogram of participants", + // We care more about tracking low quality aggregates with low participation + // Max committee sizes are: 0.5e6 vc: 244, 1e6 vc: 488 + buckets: [1, 5, 20, 50, 100, 200, 400], + }), + producedSyncContributionParticipants: register.histogram({ + name: "lodestar_produced_sync_contribution_participants", + help: "API impl produced sync contribution histogram of participants", + // We care more about tracking low quality aggregates with low participation + // Max committee sizes fixed to 512/4 = 128 + buckets: [1, 5, 20, 50, 128], + }), + producedSyncAggregateParticipants: register.histogram({ + name: "lodestar_produced_sync_aggregate_participants", + help: "API impl produced sync aggregate histogram of participants", + // We care more about tracking low quality aggregates with low participation + // Max committee sizes fixed to 512 + buckets: [1, 5, 20, 50, 100, 200, 512], + }), + }, + // Beacon state transition metrics epochTransitionTime: register.histogram({ diff --git a/packages/beacon-node/test/perf/util/bitArray.test.ts b/packages/beacon-node/test/perf/util/bitArray.test.ts index fe86aad11a09..fe421a8f9bd8 100644 --- a/packages/beacon-node/test/perf/util/bitArray.test.ts +++ b/packages/beacon-node/test/perf/util/bitArray.test.ts @@ -1,3 +1,4 @@ +import crypto from "node:crypto"; import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {BitArray} from "@chainsafe/ssz"; import {intersectUint8Arrays} from "../../../src/util/bitArray.js"; @@ -45,6 +46,18 @@ describe("Intersect BitArray vs Array+Set", () => { } }); +describe("BitArray.trueBitCount", () => { + for (const bitLen of [128, 248, 512]) { + itBench({ + id: `bitArray.getTrueBitIndexes() bitLen ${bitLen}`, + beforeEach: () => new BitArray(crypto.randomBytes(Math.ceil(bitLen / 8)), bitLen), + fn: (bitArray) => { + bitArray.getTrueBitIndexes().length; + }, + }); + } +}); + function linspace(start: number, end: number, step: number): number[] { const arr: number[] = []; for (let i = start; i < end; i += step) { From 57f13244f7d17691bd9b8c0157577a595466c44d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jul 2023 14:27:05 -0400 Subject: [PATCH 64/96] chore(deps): bump semver from 5.7.1 to 5.7.2 (#5745) Bumps [semver](https://github.com/npm/node-semver) from 5.7.1 to 5.7.2. - [Release notes](https://github.com/npm/node-semver/releases) - [Changelog](https://github.com/npm/node-semver/blob/v5.7.2/CHANGELOG.md) - [Commits](https://github.com/npm/node-semver/compare/v5.7.1...v5.7.2) --- updated-dependencies: - dependency-name: semver dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/yarn.lock b/yarn.lock index 0a18d69f71bb..4897b2e77319 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13529,9 +13529,9 @@ semver-compare@^1.0.0: integrity sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow== "semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.5.0, semver@^5.6.0: - version "5.7.1" - resolved "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz" - integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== semver@7.3.4: version "7.3.4" @@ -13540,7 +13540,7 @@ semver@7.3.4: dependencies: lru-cache "^6.0.0" -semver@7.3.8, semver@^7.3.8: +semver@7.3.8: version "7.3.8" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.8.tgz#07a78feafb3f7b32347d725e33de7e2a2df67798" integrity sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A== @@ -13548,21 +13548,14 @@ semver@7.3.8, semver@^7.3.8: lru-cache "^6.0.0" semver@^6.0.0, semver@^6.1.0, semver@^6.2.0, semver@^6.3.0: - version "6.3.0" - resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== - -semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7: - version "7.3.7" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" - integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g== - dependencies: - lru-cache "^6.0.0" - -semver@^7.5.0: - version "7.5.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" - integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.0: + version "7.5.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== dependencies: lru-cache "^6.0.0" From cdca9fbd589e450bcdb190032e97e16ea4ba0f13 Mon Sep 17 00:00:00 2001 From: g11tech Date: Sat, 15 Jul 2023 00:23:44 +0530 Subject: [PATCH 65/96] fix: lower the block production wait cutoff time for timely proposal publish (#5760) --- packages/validator/src/services/block.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/validator/src/services/block.ts b/packages/validator/src/services/block.ts index 6e8dbe120f60..2627bb1892e1 100644 --- a/packages/validator/src/services/block.ts +++ b/packages/validator/src/services/block.ts @@ -38,8 +38,12 @@ const MAX_DECIMAL_FACTOR = BigInt("100000"); /** * Cutoff time to wait for execution and builder block production apis to resolve * Post this time, race execution and builder to pick whatever resolves first + * + * Emprically the builder block resolves in ~1.5+ seconds, and executon should resolve <1 sec. + * So lowering the cutoff to 2 sec from 3 seconds to publish faster for successful proposal + * as proposals post 4 seconds into the slot seems to be not being included */ -const BLOCK_PRODUCTION_RACE_CUTOFF_MS = 3_000; +const BLOCK_PRODUCTION_RACE_CUTOFF_MS = 2_000; /** Overall timeout for execution and block production apis */ const BLOCK_PRODUCTION_RACE_TIMEOUT_MS = 12_000; From 6e014217f9df78c0916299d357588a5bbab05d88 Mon Sep 17 00:00:00 2001 From: Cayman Date: Sun, 16 Jul 2023 08:17:27 -0400 Subject: [PATCH 66/96] chore!: use node v20 throughout monorepo (#5730) * chore!: use node 20 throughout monorepo * Add yarn.lock * Revert @types/node change * Fix ts-node usage * Add missed files * Remove more ts-node-esm * Update cross-fetch dependency * Update @types/node to 20.4.2 * Use node version 20.4 in CI * Revert "Use node version 20.4 in CI" * skip tests that break in CI * Update Dockerfile Co-authored-by: Nico Flaig * Check code ECONNRESET in Lighthouse health response * Revert "Check code ECONNRESET in Lighthouse health response" * Revert "Update Dockerfile" This reverts commit e1f07be309449069312c46b696c17c8f8b9e18cb. * Revert CI from using node 20 * Revert "skip tests that break in CI" This reverts commit d296b2f1a280189761bb6175343a43362010f881. * Fix the loader for few sim tests * Update CI node version to 20 * Fix e2e test * Remove try/catch from e2e test * Consistently use node 20 in CI * Fix missed 18.x reference in types readme * Set node 20.x in readme * Use node 18 in sim tests --------- Co-authored-by: Nico Flaig Co-authored-by: Nazar Hussain --- .github/workflows/benchmark.yml | 2 +- .github/workflows/docs-check.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/publish-dev.yml | 2 +- .github/workflows/publish-rc.yml | 2 +- .github/workflows/publish-stable.yml | 2 +- .github/workflows/test-browser.yml | 2 +- .github/workflows/test-e2e.yml | 2 +- .github/workflows/test-sim-merge.yml | 2 +- .github/workflows/test-spec.yml | 2 +- .github/workflows/test.yml | 2 +- Dockerfile | 6 +-- README.md | 2 +- docs/install/source.md | 2 +- package.json | 2 +- packages/api/README.md | 2 +- packages/api/package.json | 2 +- packages/beacon-node/README.md | 2 +- packages/beacon-node/package.json | 2 +- .../test/e2e/eth1/jsonRpcHttpClient.test.ts | 23 +++++++-- packages/cli/package.json | 10 ++-- packages/light-client/package.json | 2 +- packages/params/README.md | 2 +- .../params/test/e2e/overridePreset.test.ts | 5 +- packages/params/test/e2e/setPreset.test.ts | 5 +- packages/prover/README.md | 2 +- packages/prover/package.json | 2 +- packages/reqresp/README.md | 2 +- packages/types/README.md | 2 +- packages/validator/README.md | 2 +- packages/validator/package.json | 2 +- scripts/run_e2e_env.sh | 2 +- yarn.lock | 49 +++++++++++++------ 33 files changed, 91 insertions(+), 61 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d1de888472cf..98842a38b026 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 check-latest: true - name: Node.js version id: node diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index bad86bd201b9..1556cd191b55 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 98b4ba2b56ee..cdceb49d808a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 check-latest: true - name: Node.js version id: node diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index 673b2ec6af92..a656f8562bf3 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -20,7 +20,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 registry-url: "https://registry.npmjs.org" check-latest: true - name: Node.js version diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index 07106e1945f7..005d2738b1c6 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -54,7 +54,7 @@ jobs: fetch-depth: 0 # Needs full depth for changelog generation - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 check-latest: true - name: Node.js version id: node diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 40f0cba82f78..01e222d48e72 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -60,7 +60,7 @@ jobs: fetch-depth: 0 # Needs full depth for changelog generation - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 check-latest: true - name: Node.js version id: node diff --git a/.github/workflows/test-browser.yml b/.github/workflows/test-browser.yml index c8854c1896aa..c4c478b53a07 100644 --- a/.github/workflows/test-browser.yml +++ b/.github/workflows/test-browser.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - node: [18] + node: [20] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v3 diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index d76910185489..25d49c64ad01 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - node: [18] + node: [20] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v3 diff --git a/.github/workflows/test-sim-merge.yml b/.github/workflows/test-sim-merge.yml index 60cf84560766..a47a2a07a5a1 100644 --- a/.github/workflows/test-sim-merge.yml +++ b/.github/workflows/test-sim-merge.yml @@ -29,7 +29,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 check-latest: true - name: Node.js version id: node diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 58a80543303f..eb17c2e2babf 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 check-latest: true - name: Node.js version id: node diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b95fb6a44794..22bef8d6c10a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - node: [18] + node: [20] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v3 diff --git a/Dockerfile b/Dockerfile index 346fffd117dc..5a9541e06f1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # --platform=$BUILDPLATFORM is used build javascript source with host arch # Otherwise TS builds on emulated archs and can be extremely slow (+1h) -FROM --platform=${BUILDPLATFORM:-amd64} node:18-alpine as build_src +FROM --platform=${BUILDPLATFORM:-amd64} node:20-alpine as build_src ARG COMMIT WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 && rm -rf /var/cache/apk/* @@ -21,7 +21,7 @@ RUN cd packages/cli && GIT_COMMIT=${COMMIT} yarn write-git-data # Copy built src + node_modules to build native packages for archs different than host. # Note: This step is redundant for the host arch -FROM node:18-alpine as build_deps +FROM node:20-alpine as build_deps WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 && rm -rf /var/cache/apk/* @@ -35,7 +35,7 @@ RUN cd node_modules/classic-level && yarn rebuild # Copy built src + node_modules to a new layer to prune unnecessary fs # Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020) -FROM node:18-alpine +FROM node:20-alpine WORKDIR /usr/app COPY --from=build_deps /usr/app . diff --git a/README.md b/README.md index abfcb754e26d..9eeeb12ef43e 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Ethereum Consensus Spec v1.1.10](https://img.shields.io/badge/ETH%20consensus--spec-1.1.10-blue)](https://github.com/ethereum/consensus-specs/releases/tag/v1.1.10) [![codecov](https://codecov.io/gh/ChainSafe/lodestar/branch/unstable/graph/badge.svg)](https://codecov.io/gh/ChainSafe/lodestar) ![ES Version](https://img.shields.io/badge/ES-2020-yellow) -![Node Version](https://img.shields.io/badge/node-18.15.x-green) +![Node Version](https://img.shields.io/badge/node-20.x-green) [![gitpoap badge](https://public-api.gitpoap.io/v1/repo/ChainSafe/lodestar/badge)](https://www.gitpoap.io/gh/ChainSafe/lodestar) [Lodestar](https://lodestar.chainsafe.io) is a TypeScript implementation of the [Ethereum Consensus specification](https://github.com/ethereum/consensus-specs) developed by [ChainSafe Systems](https://chainsafe.io). diff --git a/docs/install/source.md b/docs/install/source.md index 6068c5189093..4fba0a625111 100644 --- a/docs/install/source.md +++ b/docs/install/source.md @@ -2,7 +2,7 @@ ## Prerequisites -Make sure to have [Yarn installed](https://classic.yarnpkg.com/en/docs/install). It is also recommended to [install NVM (Node Version Manager)](https://github.com/nvm-sh/nvm) and use the LTS version (currently v18) of [NodeJS](https://nodejs.org/en/). +Make sure to have [Yarn installed](https://classic.yarnpkg.com/en/docs/install). It is also recommended to [install NVM (Node Version Manager)](https://github.com/nvm-sh/nvm) and use the LTS version (currently v20) of [NodeJS](https://nodejs.org/en/). !!! info diff --git a/package.json b/package.json index b4f7611a33c5..4b7d0fc5268f 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "@types/chai": "^4.3.4", "@types/chai-as-promised": "^7.1.5", "@types/mocha": "^10.0.1", - "@types/node": "^18.15.11", + "@types/node": "^20.4.2", "@types/sinon": "^10.0.13", "@types/sinon-chai": "^3.2.9", "@typescript-eslint/eslint-plugin": "6.0.0", diff --git a/packages/api/README.md b/packages/api/README.md index c8a1ebf3a31a..877e04384ee4 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -3,7 +3,7 @@ [![Discord](https://img.shields.io/discord/593655374469660673.svg?label=Discord&logo=discord)](https://discord.gg/aMxzVcr) [![ETH Beacon APIs Spec v2.1.0](https://img.shields.io/badge/ETH%20beacon--APIs-2.1.0-blue)](https://github.com/ethereum/beacon-APIs/releases/tag/v2.1.0) ![ES Version](https://img.shields.io/badge/ES-2020-yellow) -![Node Version](https://img.shields.io/badge/node-18.x-green) +![Node Version](https://img.shields.io/badge/node-20.x-green) > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project diff --git a/packages/api/package.json b/packages/api/package.json index 28cb5dc1f6c9..b925953770f1 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -75,7 +75,7 @@ "@lodestar/params": "^1.9.1", "@lodestar/types": "^1.9.1", "@lodestar/utils": "^1.9.1", - "cross-fetch": "^3.1.4", + "cross-fetch": "^4.0.0", "eventsource": "^2.0.2", "qs": "^6.11.1" }, diff --git a/packages/beacon-node/README.md b/packages/beacon-node/README.md index 67e26c73a9ff..955cfdb0e9b6 100644 --- a/packages/beacon-node/README.md +++ b/packages/beacon-node/README.md @@ -3,7 +3,7 @@ [![Discord](https://img.shields.io/discord/593655374469660673.svg?label=Discord&logo=discord)](https://discord.gg/aMxzVcr) [![Eth Consensus Spec v1.1.10](https://img.shields.io/badge/ETH%20consensus--spec-1.1.10-blue)](https://github.com/ethereum/consensus-specs/releases/tag/v1.1.10) ![ES Version](https://img.shields.io/badge/ES-2020-yellow) -![Node Version](https://img.shields.io/badge/node-18.x-green) +![Node Version](https://img.shields.io/badge/node-20.x-green) > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 73b37098d887..7b738005c1cc 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -136,7 +136,7 @@ "@types/datastore-level": "^3.0.0", "buffer-xor": "^2.0.2", "c-kzg": "^2.1.0", - "cross-fetch": "^3.1.4", + "cross-fetch": "^4.0.0", "datastore-core": "^9.1.1", "datastore-level": "^10.1.1", "deepmerge": "^4.3.1", diff --git a/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts b/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts index 9ab0068c06a0..6de1cebf6fc8 100644 --- a/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts +++ b/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts @@ -6,6 +6,10 @@ import {JsonRpcHttpClient} from "../../../src/eth1/provider/jsonRpcHttpClient.js import {getGoerliRpcUrl} from "../../testParams.js"; import {RpcPayload} from "../../../src/eth1/interface.js"; +type FetchError = { + code: string; +}; + describe("eth1 / jsonRpcHttpClient", function () { this.timeout("10 seconds"); @@ -22,6 +26,7 @@ describe("eth1 / jsonRpcHttpClient", function () { abort?: true; timeout?: number; error: any; + errorCode?: string; }[] = [ // // NOTE: This DNS query is very expensive, all cache miss. So it can timeout the tests and cause false positives // { @@ -39,7 +44,8 @@ describe("eth1 / jsonRpcHttpClient", function () { id: "Bad port", url: `http://localhost:${port + 1}`, requestListener: (req, res) => res.end(), - error: "connect ECONNREFUSED", + error: "", + errorCode: "ECONNREFUSED", }, { id: "Not a JSON RPC endpoint", @@ -122,7 +128,6 @@ describe("eth1 / jsonRpcHttpClient", function () { for (const testCase of testCases) { const {id, requestListener, abort, timeout} = testCase; - const error = testCase.error as Error; let {url, payload} = testCase; it(id, async function () { @@ -148,7 +153,13 @@ describe("eth1 / jsonRpcHttpClient", function () { const controller = new AbortController(); if (abort) setTimeout(() => controller.abort(), 50); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); - await expect(eth1JsonRpcClient.fetch(payload, {timeout})).to.be.rejectedWith(error); + await expect(eth1JsonRpcClient.fetch(payload, {timeout})).to.be.rejected.then((error) => { + if (testCase.errorCode) { + expect((error as FetchError).code).to.be.equal(testCase.errorCode); + } else { + expect((error as Error).message).to.include(testCase.error); + } + }); }); } }); @@ -210,8 +221,10 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { return true; }, }) - ).to.be.rejectedWith("connect ECONNREFUSED"); - expect(retryCount).to.be.equal(retryAttempts, "connect ECONNREFUSED should be retried before failing"); + ).to.be.rejected.then((error) => { + expect((error as FetchError).code).to.be.equal("ECONNREFUSED"); + }); + expect(retryCount).to.be.equal(retryAttempts, "code ECONNREFUSED should be retried before failing"); }); it("should retry 404", async function () { diff --git a/packages/cli/package.json b/packages/cli/package.json index af6e9e0434da..53404de0890c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -24,7 +24,7 @@ "build": "tsc -p tsconfig.build.json && yarn write-git-data", "build:release": "yarn clean && yarn run build", "build:watch": "tsc -p tsconfig.build.json --watch", - "build:refdocs": "ts-node --esm ./docsgen/index.ts docs/cli.md", + "build:refdocs": "node --loader ts-node/esm ./docsgen/index.ts docs/cli.md", "write-git-data": "node lib/util/gitData/writeGitData.js", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\" lodestar --help", "check-types": "tsc", @@ -33,10 +33,10 @@ "pretest": "yarn run check-types", "test:unit": "nyc --cache-dir .nyc_output/.cache -e .ts mocha 'test/unit/**/*.test.ts'", "test:e2e": "mocha --timeout 30000 'test/e2e/**/*.test.ts'", - "test:sim:multifork": "LODESTAR_PRESET=minimal ts-node --esm test/sim/multi_fork.test.ts", - "test:sim:endpoints": "LODESTAR_PRESET=minimal ts-node --esm test/sim/endpoints.test.ts", - "test:sim:deneb": "LODESTAR_PRESET=minimal ts-node --esm test/sim/deneb.test.ts", - "test:sim:backup_eth_provider": "LODESTAR_PRESET=minimal ts-node --esm test/sim/backup_eth_provider.test.ts", + "test:sim:multifork": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/multi_fork.test.ts", + "test:sim:endpoints": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/endpoints.test.ts", + "test:sim:deneb": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/deneb.test.ts", + "test:sim:backup_eth_provider": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/backup_eth_provider.test.ts", "test": "yarn test:unit && yarn test:e2e", "coverage": "codecov -F lodestar", "check-readme": "typescript-docs-verifier" diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 61c685517e6c..2b3779df81f4 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -73,7 +73,7 @@ "@lodestar/state-transition": "^1.9.1", "@lodestar/types": "^1.9.1", "@lodestar/utils": "^1.9.1", - "cross-fetch": "^3.1.4", + "cross-fetch": "^4.0.0", "mitt": "^3.0.0", "strict-event-emitter-types": "^2.0.0" }, diff --git a/packages/params/README.md b/packages/params/README.md index ebce8bda5b1b..bf4457146fbd 100644 --- a/packages/params/README.md +++ b/packages/params/README.md @@ -4,7 +4,7 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Eth Consensus Spec v1.1.10](https://img.shields.io/badge/ETH%20consensus--spec-1.1.10-blue)](https://github.com/ethereum/consensus-specs/releases/tag/v1.1.10) ![ES Version](https://img.shields.io/badge/ES-2020-yellow) -![Node Version](https://img.shields.io/badge/node-18.x-green) +![Node Version](https://img.shields.io/badge/node-20.x-green) > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project diff --git a/packages/params/test/e2e/overridePreset.test.ts b/packages/params/test/e2e/overridePreset.test.ts index 61483e2cee91..e16dd97a08ef 100644 --- a/packages/params/test/e2e/overridePreset.test.ts +++ b/packages/params/test/e2e/overridePreset.test.ts @@ -18,18 +18,17 @@ const exec = util.promisify(child.exec); // Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules // eslint-disable-next-line @typescript-eslint/naming-convention const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const tsNodeBinary = path.join(__dirname, "../../../../node_modules/.bin/ts-node-esm"); describe("Override preset", function () { // Allow time for ts-node to compile Typescript source this.timeout(30_000); it("Should correctly override preset", async () => { - await exec(`${tsNodeBinary} ${path.join(__dirname, scriptNames.ok)}`); + await exec(`node --loader ts-node/esm ${path.join(__dirname, scriptNames.ok)}`); }); it("Should throw trying to override preset in the wrong order", async () => { - await expect(exec(`${tsNodeBinary} ${path.join(__dirname, scriptNames.error)}`)).to.be.rejectedWith( + await expect(exec(`node --loader ts-node/esm ${path.join(__dirname, scriptNames.error)}`)).to.be.rejectedWith( "Lodestar preset is already frozen" ); }); diff --git a/packages/params/test/e2e/setPreset.test.ts b/packages/params/test/e2e/setPreset.test.ts index d2268d32fe51..2b7ff271cd94 100644 --- a/packages/params/test/e2e/setPreset.test.ts +++ b/packages/params/test/e2e/setPreset.test.ts @@ -18,18 +18,17 @@ const exec = util.promisify(child.exec); // Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules // eslint-disable-next-line @typescript-eslint/naming-convention const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const tsNodeBinary = path.join(__dirname, "../../../../node_modules/.bin/ts-node-esm"); describe("setPreset", function () { // Allow time for ts-node to compile Typescript source this.timeout(30_000); it("Should correctly set preset", async () => { - await exec(`${tsNodeBinary} ${path.join(__dirname, scriptNames.ok)}`); + await exec(`node --loader ts-node/esm ${path.join(__dirname, scriptNames.ok)}`); }); it("Should throw trying to set preset in the wrong order", async () => { - await expect(exec(`${tsNodeBinary} ${path.join(__dirname, scriptNames.error)}`)).to.be.rejectedWith( + await expect(exec(`node --loader ts-node/esm ${path.join(__dirname, scriptNames.error)}`)).to.be.rejectedWith( "Lodestar preset is already frozen" ); }); diff --git a/packages/prover/README.md b/packages/prover/README.md index c4df66751e51..46c733c9715d 100644 --- a/packages/prover/README.md +++ b/packages/prover/README.md @@ -3,7 +3,7 @@ [![Discord](https://img.shields.io/discord/593655374469660673.svg?label=Discord&logo=discord)](https://discord.gg/aMxzVcr) [![ETH Beacon APIs Spec v2.1.0](https://img.shields.io/badge/ETH%20beacon--APIs-2.1.0-blue)](https://github.com/ethereum/beacon-APIs/releases/tag/v2.1.0) ![ES Version](https://img.shields.io/badge/ES-2020-yellow) -![Node Version](https://img.shields.io/badge/node-18.x-green) +![Node Version](https://img.shields.io/badge/node-20.x-green) > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project diff --git a/packages/prover/package.json b/packages/prover/package.json index 490984014d4c..e6a9844097c2 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -55,7 +55,7 @@ "test:browsers": "yarn karma start karma.config.cjs", "test:e2e": "LODESTAR_PRESET=minimal mocha 'test/e2e/**/*.test.ts'", "check-readme": "typescript-docs-verifier", - "generate-fixtures": "npx ts-node --esm scripts/generate_fixtures.ts" + "generate-fixtures": "node --loader ts-node/esm scripts/generate_fixtures.ts" }, "dependencies": { "@ethereumjs/block": "^4.2.2", diff --git a/packages/reqresp/README.md b/packages/reqresp/README.md index ecb29a7ff2ad..a363dfc65388 100644 --- a/packages/reqresp/README.md +++ b/packages/reqresp/README.md @@ -3,7 +3,7 @@ [![Discord](https://img.shields.io/discord/593655374469660673.svg?label=Discord&logo=discord)](https://discord.gg/aMxzVcr) [![ETH Beacon APIs Spec v2.1.0](https://img.shields.io/badge/ETH%20beacon--APIs-2.1.0-blue)](https://github.com/ethereum/beacon-APIs/releases/tag/v2.1.0) ![ES Version](https://img.shields.io/badge/ES-2020-yellow) -![Node Version](https://img.shields.io/badge/node-18.x-green) +![Node Version](https://img.shields.io/badge/node-20.x-green) > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project diff --git a/packages/types/README.md b/packages/types/README.md index 0c668490bb18..9cd1f020a44d 100644 --- a/packages/types/README.md +++ b/packages/types/README.md @@ -4,7 +4,7 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Eth Consensus Spec v1.1.10](https://img.shields.io/badge/ETH%20consensus--spec-1.1.10-blue)](https://github.com/ethereum/consensus-specs/releases/tag/v1.1.10) ![ES Version](https://img.shields.io/badge/ES-2020-yellow) -![Node Version](https://img.shields.io/badge/node-18.x-green) +![Node Version](https://img.shields.io/badge/node-20.x-green) > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project diff --git a/packages/validator/README.md b/packages/validator/README.md index 84301ca7efa5..d9ae22a01f04 100644 --- a/packages/validator/README.md +++ b/packages/validator/README.md @@ -3,7 +3,7 @@ [![Discord](https://img.shields.io/discord/593655374469660673.svg?label=Discord&logo=discord)](https://discord.gg/aMxzVcr) [![Eth Consensus Spec v1.1.10](https://img.shields.io/badge/ETH%20consensus--spec-1.1.10-blue)](https://github.com/ethereum/consensus-specs/releases/tag/v1.1.10) ![ES Version](https://img.shields.io/badge/ES-2020-yellow) -![Node Version](https://img.shields.io/badge/node-18.x-green) +![Node Version](https://img.shields.io/badge/node-20.x-green) > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project diff --git a/packages/validator/package.json b/packages/validator/package.json index 1b20eb5db607..4a749792a2f0 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -58,7 +58,7 @@ "@lodestar/types": "^1.9.1", "@lodestar/utils": "^1.9.1", "bigint-buffer": "^1.1.5", - "cross-fetch": "^3.1.4", + "cross-fetch": "^4.0.0", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { diff --git a/scripts/run_e2e_env.sh b/scripts/run_e2e_env.sh index 180cda203d94..08f1680b30c6 100755 --- a/scripts/run_e2e_env.sh +++ b/scripts/run_e2e_env.sh @@ -3,7 +3,7 @@ function start_app() { mkdir -p test-logs/e2e-test-env export LODESTAR_PRESET=minimal - nohup npx ts-node --esm packages/cli/test/scripts/e2e_test_env.ts > test-logs/e2e-test-env/simulation.out 2>&1 & + nohup node --loader ts-node/esm packages/cli/test/scripts/e2e_test_env.ts > test-logs/e2e-test-env/simulation.out 2>&1 & echo $! > test-logs/e2e-test-env/simulation.pid echo "Wait for the node to be ready" npx wait-port -t 60000 0.0.0.0:5001 diff --git a/yarn.lock b/yarn.lock index 4897b2e77319..d5317b94ddb2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3696,10 +3696,10 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.45.tgz#155b13a33c665ef2b136f7f245fa525da419e810" integrity sha512-3rKg/L5x0rofKuuUt5zlXzOnKyIHXmIu5R8A0TuNDMF2062/AOIDBciFIjToLEJ/9F9DzkHNot+BpNsMI1OLdQ== -"@types/node@^18.15.11": - version "18.15.11" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.15.11.tgz#b3b790f09cb1696cffcec605de025b088fa4225f" - integrity sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q== +"@types/node@^20.4.2": + version "20.4.2" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.4.2.tgz#129cc9ae69f93824f92fac653eebfb4812ab4af9" + integrity sha512-Dd0BYtWgnWJKwO1jkmTrzofjK2QXXcai0dmtzvIBhcA+RsG5h8R3xlyta0kGOZRNfL9GuRtb1knmPEhQrePCEw== "@types/normalize-package-data@^2.4.0": version "2.4.1" @@ -6128,6 +6128,13 @@ cross-fetch@^3.1.4: dependencies: node-fetch "2.6.7" +cross-fetch@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-4.0.0.tgz#f037aef1580bb3a1a35164ea2a848ba81b445983" + integrity sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g== + dependencies: + node-fetch "^2.6.12" + cross-spawn@^6.0.5: version "6.0.5" resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz" @@ -10558,11 +10565,6 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -lru-cache@^7.14.1: - version "7.18.3" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89" - integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA== - lru-cache@^7.4.4, lru-cache@^7.5.1: version "7.14.0" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.14.0.tgz#21be64954a4680e303a09e9468f880b98a0b3c7f" @@ -10573,6 +10575,11 @@ lru-cache@^7.7.1: resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.10.1.tgz#db577f42a94c168f676b638d15da8fb073448cab" integrity sha512-BQuhQxPuRl79J5zSXRP+uNzPOyZw2oFI9JLRQ80XswSvg21KMKNtQza9eF42rfI/3Z40RvzBdXgziEkudzjo8A== +"lru-cache@^9.1.1 || ^10.0.0": + version "10.0.0" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.0.0.tgz#b9e2a6a72a129d81ab317202d93c7691df727e61" + integrity sha512-svTf/fzsKHffP42sujkO/Rjs37BCIsQVRCeNYIm9WN8rgT7ffoUnRtZCqU+6BqcSBdv8gwJeTz8knJpgACeQMw== + make-dir@3.1.0, make-dir@^3.0.0, make-dir@^3.0.2: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" @@ -11020,11 +11027,16 @@ minipass@^3.1.6: dependencies: yallist "^4.0.0" -minipass@^4.0.0, minipass@^4.0.2, minipass@^4.2.4: +minipass@^4.0.0, minipass@^4.2.4: version "4.2.5" resolved "https://registry.yarnpkg.com/minipass/-/minipass-4.2.5.tgz#9e0e5256f1e3513f8c34691dd68549e85b2c8ceb" integrity sha512-+yQl7SX3bIT83Lhb4BVorMAHVuqsskxRdlmO9kTpyukp8vsm2Sn/fUOV9xlnG8/a5JsypJzap21lz/y3FBMJ8Q== +"minipass@^5.0.0 || ^6.0.2 || ^7.0.0": + version "7.0.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.2.tgz#58a82b7d81c7010da5bd4b2c0c85ac4b4ec5131e" + integrity sha512-eL79dXrE1q9dBbDCLg7xfn/vl7MS4F1gvJAgjJrQli/jbQWdUttuVawphqpffoIYfRdq78LHx6GP4bU/EQ2ATA== + minizlib@^1.3.3: version "1.3.3" resolved "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz" @@ -11354,6 +11366,13 @@ node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: resolved "https://registry.npmjs.org/@achingbrain/node-fetch/-/node-fetch-2.6.7.tgz" integrity sha512-iTASGs+HTFK5E4ZqcMsHmeJ4zodyq8L38lZV33jwqcBJYoUt3HjN4+ot+O9/0b+ke8ddE7UgOtVuZN/OkV19/g== +node-fetch@^2.6.12: + version "2.6.12" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.12.tgz#02eb8e22074018e3d5a83016649d04df0e348fba" + integrity sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g== + dependencies: + whatwg-url "^5.0.0" + node-fetch@^2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.9.tgz#7c7f744b5cc6eb5fd404e0c7a9fec630a55657e6" @@ -12439,12 +12458,12 @@ path-parse@^1.0.5, path-parse@^1.0.6, path-parse@^1.0.7: integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== path-scurry@^1.6.1: - version "1.6.3" - resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.6.3.tgz#4eba7183d64ef88b63c7d330bddc3ba279dc6c40" - integrity sha512-RAmB+n30SlN+HnNx6EbcpoDy9nwdpcGPnEKrJnu6GZoDWBdIjo1UQMVtW2ybtC7LC2oKLcMq8y5g8WnKLiod9g== + version "1.10.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.10.1.tgz#9ba6bf5aa8500fe9fd67df4f0d9483b2b0bfc698" + integrity sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ== dependencies: - lru-cache "^7.14.1" - minipass "^4.0.2" + lru-cache "^9.1.1 || ^10.0.0" + minipass "^5.0.0 || ^6.0.2 || ^7.0.0" path-to-regexp@0.1.7: version "0.1.7" From e3eb0557ee9c70d08e50c4d4d2c3265ab5b96815 Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Mon, 17 Jul 2023 18:29:52 +0700 Subject: [PATCH 67/96] fix: correct subscribedToNetworkEvents flag of UnknownBlockSync (#5764) fix: UnknownBlockSync subscribedToNetworkEvents flag --- packages/beacon-node/src/sync/unknownBlock.ts | 6 +- .../test/unit/sync/unknownBlock.test.ts | 65 ++++++++++++++++++- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/packages/beacon-node/src/sync/unknownBlock.ts b/packages/beacon-node/src/sync/unknownBlock.ts index b41d0c4cb06e..82654f501346 100644 --- a/packages/beacon-node/src/sync/unknownBlock.ts +++ b/packages/beacon-node/src/sync/unknownBlock.ts @@ -57,21 +57,23 @@ export class UnknownBlockSync { if (!this.opts?.disableUnknownBlockSync) { // cannot chain to the above if or the log will be incorrect if (!this.subscribedToNetworkEvents) { - this.logger.debug("UnknownBlockSync enabled."); + this.logger.verbose("UnknownBlockSync enabled."); this.network.events.on(NetworkEvent.unknownBlock, this.onUnknownBlock); this.network.events.on(NetworkEvent.unknownBlockParent, this.onUnknownParent); this.network.events.on(NetworkEvent.peerConnected, this.triggerUnknownBlockSearch); this.subscribedToNetworkEvents = true; } } else { - this.logger.debug("UnknownBlockSync disabled."); + this.logger.verbose("UnknownBlockSync disabled by disableUnknownBlockSync option."); } } unsubscribeFromNetwork(): void { + this.logger.verbose("UnknownBlockSync disabled."); this.network.events.off(NetworkEvent.unknownBlock, this.onUnknownBlock); this.network.events.off(NetworkEvent.unknownBlockParent, this.onUnknownParent); this.network.events.off(NetworkEvent.peerConnected, this.triggerUnknownBlockSearch); + this.subscribedToNetworkEvents = false; } close(): void { diff --git a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts index 3bbc72e98f52..40e10fed51d4 100644 --- a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts +++ b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts @@ -1,12 +1,13 @@ +import EventEmitter from "node:events"; import {expect} from "chai"; -import sinon from "sinon"; +import sinon, {SinonStubbedInstance} from "sinon"; import {toHexString} from "@chainsafe/ssz"; import {config as minimalConfig} from "@lodestar/config/default"; import {createChainForkConfig} from "@lodestar/config"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {ssz} from "@lodestar/types"; import {notNullish, sleep} from "@lodestar/utils"; -import {IBeaconChain} from "../../../src/chain/index.js"; +import {BeaconChain, IBeaconChain} from "../../../src/chain/index.js"; import {INetwork, NetworkEvent, NetworkEventBus, PeerAction} from "../../../src/network/index.js"; import {UnknownBlockSync} from "../../../src/sync/unknownBlock.js"; import {testLogger} from "../../utils/logger.js"; @@ -18,7 +19,7 @@ import {BlockError, BlockErrorCode} from "../../../src/chain/errors/blockError.j import {defaultSyncOptions} from "../../../src/sync/options.js"; import {ZERO_HASH} from "../../../src/constants/constants.js"; -describe("sync / UnknownBlockSync", () => { +describe("sync by UnknownBlockSync", () => { const logger = testLogger(); const sandbox = sinon.createSandbox(); const slotSec = 0.3; @@ -230,3 +231,61 @@ describe("sync / UnknownBlockSync", () => { }); } }); + +describe("UnknownBlockSync", function () { + const sandbox = sinon.createSandbox(); + let network: INetwork; + let chain: SinonStubbedInstance & IBeaconChain; + const logger = testLogger(); + let service: UnknownBlockSync; + + beforeEach(() => { + network = { + events: new NetworkEventBus(), + } as Partial as INetwork; + chain = sandbox.createStubInstance(BeaconChain); + }); + + afterEach(() => { + sandbox.restore(); + }); + + const testCases: {actions: boolean[]; expected: boolean}[] = [ + // true = subscribe, false = unsubscribe + // expected = isSubscribed + {actions: [false, true], expected: true}, + {actions: [false, true, true], expected: true}, + {actions: [true, false, true], expected: true}, + {actions: [true, true, true], expected: true}, + {actions: [true, false, false, true], expected: true}, + {actions: [true, false], expected: false}, + {actions: [true, false, false], expected: false}, + ]; + + describe("subscribe and unsubscribe multiple times", () => { + for (const {actions, expected} of testCases) { + const testName = actions.map((action) => (action ? "subscribe" : "unsubscribe")).join(" - "); + it(testName, () => { + const events = network.events as EventEmitter; + service = new UnknownBlockSync(minimalConfig, network, chain, logger, null, defaultSyncOptions); + for (const action of actions) { + if (action) { + service.subscribeToNetwork(); + } else { + service.unsubscribeFromNetwork(); + } + } + + if (expected) { + expect(events.listenerCount(NetworkEvent.unknownBlock)).to.be.equal(1); + expect(events.listenerCount(NetworkEvent.unknownBlockParent)).to.be.equal(1); + expect(service.isSubscribedToNetwork()).to.be.true; + } else { + expect(events.listenerCount(NetworkEvent.unknownBlock)).to.be.equal(0); + expect(events.listenerCount(NetworkEvent.unknownBlockParent)).to.be.equal(0); + expect(service.isSubscribedToNetwork()).to.be.false; + } + }); + } + }); +}); From ec8153153c83d082cb89fca682d502e709e3f415 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 17 Jul 2023 09:31:03 -0400 Subject: [PATCH 68/96] feat: add IPv6 support (#5758) * feat: add IPv6 support * fix: ip -> ip4 * fix: add parens * fix: update cli flag descriptions --- packages/beacon-node/package.json | 2 +- .../beacon-node/src/network/discv5/index.ts | 2 +- .../beacon-node/src/network/discv5/types.ts | 18 +++- .../beacon-node/src/network/discv5/worker.ts | 17 +++- .../beacon-node/test/e2e/network/mdns.test.ts | 4 +- .../test/unit/network/util.test.ts | 8 +- packages/cli/package.json | 2 +- .../cli/src/cmds/beacon/initPeerIdAndEnr.ts | 50 +++++----- .../src/options/beaconNodeOptions/network.ts | 92 +++++++++++++++++-- .../unit/options/beaconNodeOptions.test.ts | 4 +- yarn.lock | 8 +- 11 files changed, 160 insertions(+), 47 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 7b738005c1cc..25a776366ea4 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -98,7 +98,7 @@ "@chainsafe/as-sha256": "^0.3.1", "@chainsafe/bls": "7.1.1", "@chainsafe/blst": "^0.2.9", - "@chainsafe/discv5": "^4.0.0", + "@chainsafe/discv5": "^5.0.0", "@chainsafe/libp2p-gossipsub": "^9.1.0", "@chainsafe/libp2p-noise": "^12.0.1", "@chainsafe/persistent-merkle-tree": "^0.5.0", diff --git a/packages/beacon-node/src/network/discv5/index.ts b/packages/beacon-node/src/network/discv5/index.ts index 611898973d32..089e8813f5eb 100644 --- a/packages/beacon-node/src/network/discv5/index.ts +++ b/packages/beacon-node/src/network/discv5/index.ts @@ -43,7 +43,7 @@ export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter[0]["config"]; +type BindAddrs = + | { + ip4: string; + ip6?: string; + } + | { + ip4?: string; + ip6: string; + } + | { + ip4: string; + ip6: string; + }; + export type LodestarDiscv5Opts = { config?: Discv5Config; enr: string; - bindAddr: string; + bindAddrs: BindAddrs; bootEnrs: string[]; }; @@ -17,7 +31,7 @@ export type LodestarDiscv5Opts = { export interface Discv5WorkerData { enr: string; peerIdProto: Uint8Array; - bindAddr: string; + bindAddrs: BindAddrs; config: Discv5Config; bootEnrs: string[]; metrics: boolean; diff --git a/packages/beacon-node/src/network/discv5/worker.ts b/packages/beacon-node/src/network/discv5/worker.ts index 282128c27614..9b7cb1a6dce3 100644 --- a/packages/beacon-node/src/network/discv5/worker.ts +++ b/packages/beacon-node/src/network/discv5/worker.ts @@ -1,6 +1,6 @@ import worker from "node:worker_threads"; import {createFromProtobuf} from "@libp2p/peer-id-factory"; -import {multiaddr} from "@multiformats/multiaddr"; +import {Multiaddr, multiaddr} from "@multiformats/multiaddr"; import {Gauge} from "prom-client"; import {expose} from "@chainsafe/threads/worker"; import {Observable, Subject} from "@chainsafe/threads/observable"; @@ -48,7 +48,10 @@ const config = createBeaconConfig(workerData.chainConfig, workerData.genesisVali const discv5 = Discv5.create({ enr: SignableENR.decodeTxt(workerData.enr, keypair), peerId, - multiaddr: multiaddr(workerData.bindAddr), + bindAddrs: { + ip4: (workerData.bindAddrs.ip4 ? multiaddr(workerData.bindAddrs.ip4) : undefined) as Multiaddr, + ip6: workerData.bindAddrs.ip6 ? multiaddr(workerData.bindAddrs.ip6) : undefined, + }, config: workerData.config, metricsRegistry, }); @@ -105,8 +108,12 @@ const module: Discv5WorkerApi = { expose(module); -logger.info("discv5 worker started", { +const logData: Record = { peerId: peerId.toString(), - listenAddr: workerData.bindAddr, initialENR: workerData.enr, -}); +}; + +if (workerData.bindAddrs.ip4) logData.bindAddr4 = workerData.bindAddrs.ip4; +if (workerData.bindAddrs.ip6) logData.bindAddr6 = workerData.bindAddrs.ip6; + +logger.info("discv5 worker started", logData); diff --git a/packages/beacon-node/test/e2e/network/mdns.test.ts b/packages/beacon-node/test/e2e/network/mdns.test.ts index fb38f363cfde..0b71ce47fb44 100644 --- a/packages/beacon-node/test/e2e/network/mdns.test.ts +++ b/packages/beacon-node/test/e2e/network/mdns.test.ts @@ -53,7 +53,9 @@ describe.skip("mdns", function () { discv5FirstQueryDelayMs: 0, discv5: { enr: enr.encodeTxt(), - bindAddr: bindAddrUdp, + bindAddrs: { + ip4: bindAddrUdp, + }, bootEnrs: [], }, }; diff --git a/packages/beacon-node/test/unit/network/util.test.ts b/packages/beacon-node/test/unit/network/util.test.ts index 40e05e012252..1ee056ec0786 100644 --- a/packages/beacon-node/test/unit/network/util.test.ts +++ b/packages/beacon-node/test/unit/network/util.test.ts @@ -51,7 +51,9 @@ describe("createNodeJsLibp2p", () => { connectToDiscv5Bootnodes: true, discv5: { enr: SignableENR.createV4(keypair).encodeTxt(), - bindAddr: "/ip4/127.0.0.1/udp/0", + bindAddrs: { + ip4: "/ip4/127.0.0.1/udp/0", + }, bootEnrs: enrWithTcp, }, bootMultiaddrs, @@ -81,7 +83,9 @@ describe("createNodeJsLibp2p", () => { connectToDiscv5Bootnodes: true, discv5: { enr: SignableENR.createV4(keypair).encodeTxt(), - bindAddr: "/ip4/127.0.0.1/udp/0", + bindAddrs: { + ip4: "/ip4/127.0.0.1/udp/0", + }, bootEnrs: enrWithoutTcp, }, bootMultiaddrs, diff --git a/packages/cli/package.json b/packages/cli/package.json index 53404de0890c..a82e8b97be8d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -57,7 +57,7 @@ "@chainsafe/bls-keygen": "^0.3.0", "@chainsafe/bls-keystore": "^2.0.0", "@chainsafe/blst": "^0.2.9", - "@chainsafe/discv5": "^4.0.0", + "@chainsafe/discv5": "^5.0.0", "@chainsafe/ssz": "^0.10.2", "@chainsafe/threads": "^1.11.1", "@libp2p/crypto": "^1.0.0", diff --git a/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts b/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts index 989aa2f89f63..3c7053190335 100644 --- a/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts +++ b/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts @@ -8,7 +8,7 @@ import {createKeypairFromPeerId, SignableENR} from "@chainsafe/discv5"; import {Logger} from "@lodestar/utils"; import {exportToJSON, readPeerId} from "../../config/index.js"; import {writeFile600Perm} from "../../util/file.js"; -import {defaultP2pPort} from "../../options/beaconNodeOptions/network.js"; +import {parseListenArgs} from "../../options/beaconNodeOptions/network.js"; import {BeaconArgs} from "./options.js"; /** @@ -53,27 +53,17 @@ export function isLocalMultiAddr(multiaddr: Multiaddr | undefined): boolean { return false; } -export function clearMultiaddrUDP(enr: SignableENR): void { - // enr.multiaddrUDP = undefined in new version - enr.delete("ip"); - enr.delete("udp"); - enr.delete("ip6"); - enr.delete("udp6"); -} - export function overwriteEnrWithCliArgs(enr: SignableENR, args: BeaconArgs, logger: Logger): void { - // TODO: Not sure if we should propagate port/defaultP2pPort options to the ENR - enr.tcp = args["enr.tcp"] ?? args.port ?? defaultP2pPort; - const udpPort = args["enr.udp"] ?? args.discoveryPort ?? args.port ?? defaultP2pPort; - if (udpPort != null) enr.udp = udpPort; - if (args["enr.ip"] != null) enr.ip = args["enr.ip"]; - if (args["enr.ip6"] != null) enr.ip6 = args["enr.ip6"]; - if (args["enr.tcp6"] != null) enr.tcp6 = args["enr.tcp6"]; - if (args["enr.udp6"] != null) enr.udp6 = args["enr.udp6"]; + const {listenAddress, port, discoveryPort, listenAddress6, port6, discoveryPort6} = parseListenArgs(args); + enr.ip = args["enr.ip"] ?? listenAddress; + enr.tcp = args["enr.tcp"] ?? port; + enr.udp = args["enr.udp"] ?? discoveryPort; + enr.ip6 = args["enr.ip6"] ?? listenAddress6; + enr.tcp6 = args["enr.tcp6"] ?? port6; + enr.udp6 = args["enr.udp6"] ?? discoveryPort6; - const udpMultiaddr = enr.getLocationMultiaddr("udp"); - if (udpMultiaddr) { - const isLocal = isLocalMultiAddr(udpMultiaddr); + function testMultiaddrForLocal(mu: Multiaddr, ip4: boolean): void { + const isLocal = isLocalMultiAddr(mu); if (args.nat) { if (isLocal) { logger.warn("--nat flag is set with no purpose"); @@ -81,12 +71,28 @@ export function overwriteEnrWithCliArgs(enr: SignableENR, args: BeaconArgs, logg } else { if (!isLocal) { logger.warn( - "Configured ENR IP address is not local, clearing ENR IP and UDP. Set the --nat flag to prevent this" + `Configured ENR ${ip4 ? "IPv4" : "IPv6"} address is not local, clearing ENR ${ip4 ? "ip" : "ip6"} and ${ + ip4 ? "udp" : "udp6" + }. Set the --nat flag to prevent this` ); - clearMultiaddrUDP(enr); + if (ip4) { + enr.delete("ip"); + enr.delete("udp"); + } else { + enr.delete("ip6"); + enr.delete("udp6"); + } } } } + const udpMultiaddr4 = enr.getLocationMultiaddr("udp4"); + if (udpMultiaddr4) { + testMultiaddrForLocal(udpMultiaddr4, true); + } + const udpMultiaddr6 = enr.getLocationMultiaddr("udp6"); + if (udpMultiaddr6) { + testMultiaddrForLocal(udpMultiaddr6, false); + } } /** diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index af6f27884187..6f30c3aabe76 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -1,14 +1,19 @@ +import {multiaddr} from "@multiformats/multiaddr"; import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; import {CliCommandOptions, YargsError} from "../../util/index.js"; const defaultListenAddress = "0.0.0.0"; export const defaultP2pPort = 9000; +export const defaultP2pPort6 = 9090; export type NetworkArgs = { discv5?: boolean; listenAddress?: string; port?: number; discoveryPort?: number; + listenAddress6?: string; + port6?: number; + discoveryPort6?: number; bootnodes?: string[]; targetPeers?: number; deterministicLongLivedAttnets?: boolean; @@ -38,10 +43,59 @@ export type NetworkArgs = { "network.rateTrackerTimeoutMs"?: number; }; +function validateMultiaddrArg>(args: T, key: keyof T): void { + if (args[key]) { + try { + multiaddr(args[key]); + } catch (e) { + throw new YargsError(`Invalid ${key as string}: ${(e as Error).message}`); + } + } +} + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function parseListenArgs(args: NetworkArgs) { + // If listenAddress is explicitly set, use it + // If listenAddress6 is not set, use defaultListenAddress + const listenAddress = args.listenAddress ?? (args.listenAddress6 ? undefined : defaultListenAddress); + const port = listenAddress ? args.port ?? defaultP2pPort : undefined; + const discoveryPort = listenAddress ? args.discoveryPort ?? args.port ?? defaultP2pPort : undefined; + + // Only use listenAddress6 if it is explicitly set + const listenAddress6 = args.listenAddress6; + const port6 = listenAddress6 ? args.port6 ?? defaultP2pPort6 : undefined; + const discoveryPort6 = listenAddress6 ? args.discoveryPort6 ?? args.port6 ?? defaultP2pPort6 : undefined; + + return {listenAddress, port, discoveryPort, listenAddress6, port6, discoveryPort6}; +} + export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { - const listenAddress = args.listenAddress || defaultListenAddress; - const udpPort = args.discoveryPort ?? args.port ?? defaultP2pPort; - const tcpPort = args.port ?? defaultP2pPort; + const {listenAddress, port, discoveryPort, listenAddress6, port6, discoveryPort6} = parseListenArgs(args); + // validate ip, ip6, ports + const muArgs = { + listenAddress: listenAddress ? `/ip4/${listenAddress}` : undefined, + port: listenAddress ? `/tcp/${port}` : undefined, + discoveryPort: listenAddress ? `/udp/${discoveryPort}` : undefined, + listenAddress6: listenAddress6 ? `/ip6/${listenAddress6}` : undefined, + port6: listenAddress6 ? `/tcp/${port6}` : undefined, + discoveryPort6: listenAddress6 ? `/udp/${discoveryPort6}` : undefined, + }; + + for (const key of [ + "listenAddress", + "port", + "discoveryPort", + "listenAddress6", + "port6", + "discoveryPort6", + ] as (keyof typeof muArgs)[]) { + validateMultiaddrArg(muArgs, key); + } + + const bindMu = listenAddress ? `${muArgs.listenAddress}${muArgs.discoveryPort}` : undefined; + const localMu = listenAddress ? `${muArgs.listenAddress}${muArgs.port}` : undefined; + const bindMu6 = listenAddress6 ? `${muArgs.listenAddress6}${muArgs.discoveryPort6}` : undefined; + const localMu6 = listenAddress6 ? `${muArgs.listenAddress6}${muArgs.port6}` : undefined; const targetPeers = args["targetPeers"]; const maxPeers = args["network.maxPeers"] ?? (targetPeers !== undefined ? Math.floor(targetPeers * 1.1) : undefined); @@ -54,7 +108,10 @@ export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { discv5: enableDiscv5 ? { config: {}, - bindAddr: `/ip4/${listenAddress}/udp/${udpPort}`, + bindAddrs: { + ip4: bindMu as string, + ip6: bindMu6, + }, // TODO: Okay to set to empty array? bootEnrs: args["bootnodes"] ?? [], // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any @@ -63,7 +120,7 @@ export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { : null, maxPeers: maxPeers ?? defaultOptions.network.maxPeers, targetPeers: targetPeers ?? defaultOptions.network.targetPeers, - localMultiaddrs: [`/ip4/${listenAddress}/tcp/${tcpPort}`], + localMultiaddrs: [localMu, localMu6].filter(Boolean) as string[], deterministicLongLivedAttnets: args["deterministicLongLivedAttnets"], subscribeAllSubnets: args["subscribeAllSubnets"], disablePeerScoring: args["disablePeerScoring"], @@ -93,13 +150,13 @@ export const options: CliCommandOptions = { listenAddress: { type: "string", - description: "The address to listen for p2p UDP and TCP connections", + description: "The IPv4 address to listen for p2p UDP and TCP connections", defaultDescription: defaultListenAddress, group: "network", }, port: { - description: "The TCP/UDP port to listen on. The UDP port can be modified by the --discovery-port flag.", + description: "The TCP/UDP port to listen on. The UDP port can be modified by the --discoveryPort flag.", type: "number", // TODO: Derive from BeaconNode defaults defaultDescription: String(defaultP2pPort), @@ -113,6 +170,27 @@ export const options: CliCommandOptions = { group: "network", }, + listenAddress6: { + type: "string", + description: "The IPv6 address to listen for p2p UDP and TCP connections", + group: "network", + }, + + port6: { + description: "The TCP/UDP port to listen on. The UDP port can be modified by the --discoveryPort6 flag.", + type: "number", + // TODO: Derive from BeaconNode defaults + defaultDescription: String(defaultP2pPort6), + group: "network", + }, + + discoveryPort6: { + description: "The UDP port that discovery will listen on. Defaults to `port6`", + type: "number", + defaultDescription: "`port6`", + group: "network", + }, + bootnodes: { type: "array", description: "Bootnodes for discv5 discovery", diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index acf279a9a160..ee694bc2f173 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -166,7 +166,9 @@ describe("options / beaconNodeOptions", () => { network: { discv5: { config: {}, - bindAddr: "/ip4/127.0.0.1/udp/9002", + bindAddrs: { + ip4: "/ip4/127.0.0.1/udp/9002", + }, bootEnrs: ["enr:-somedata"], }, maxPeers: 30, diff --git a/yarn.lock b/yarn.lock index d5317b94ddb2..f8d1f788b660 100644 --- a/yarn.lock +++ b/yarn.lock @@ -505,10 +505,10 @@ node-fetch "^2.6.1" node-gyp "^8.4.0" -"@chainsafe/discv5@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@chainsafe/discv5/-/discv5-4.0.0.tgz#41b9876ce0d209a4abcf844cfcdb6c4c8c338fe5" - integrity sha512-4dsfSAAa2rSRgrYM7YdvJRYNMSZ0NZzzBJw7e+PQMURvCZBRtDxh51/WS0qFX2sFrmjkT+i+xmLD8EsS4P5pUw== +"@chainsafe/discv5@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@chainsafe/discv5/-/discv5-5.0.0.tgz#d8d4eadc0ce7f649d5c1b141bb0d51efbfca4c6d" + integrity sha512-e+TbRrs1wukZgVmFuQL4tm0FRqSFRWlV7+MEbApbIenzFI7Pds1B4U5CDUFF8UE9QlkY5GEI/vXkT480Rhn6Rg== dependencies: "@libp2p/crypto" "^1.0.0" "@libp2p/interface-peer-discovery" "^2.0.0" From 5116493e87a9fe2102ebc7a3ff77677c90da3ddf Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 18 Jul 2023 14:44:28 +0200 Subject: [PATCH 69/96] fix: prevent updating execution engine state during shutdown (#5770) --- packages/beacon-node/src/execution/engine/http.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index 60e66aba2d85..0a6e18b9ee53 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -1,10 +1,11 @@ import {Root, RootHex, allForks, Wei} from "@lodestar/types"; import {SLOTS_PER_EPOCH, ForkName, ForkSeq} from "@lodestar/params"; import {Logger} from "@lodestar/logger"; +import {isErrorAborted} from "@lodestar/utils"; import {ErrorJsonRpcResponse, HttpRpcError} from "../../eth1/provider/jsonRpcHttpClient.js"; import {IJsonRpcHttpClient, ReqOpts} from "../../eth1/provider/jsonRpcHttpClient.js"; import {Metrics} from "../../metrics/index.js"; -import {JobItemQueue} from "../../util/queue/index.js"; +import {JobItemQueue, isQueueErrorAborted} from "../../util/queue/index.js"; import {EPOCHS_PER_BATCH} from "../../sync/constants.js"; import {numToQuantity} from "../../eth1/provider/utils.js"; import {IJson, RpcPayload} from "../../eth1/interface.js"; @@ -130,7 +131,9 @@ export class ExecutionEngineHttp implements IExecutionEngine { this.updateEngineState(ExecutionEngineState.ONLINE); return res; } catch (err) { - this.updateEngineState(getExecutionEngineState({payloadError: err})); + if (!isErrorAborted(err)) { + this.updateEngineState(getExecutionEngineState({payloadError: err})); + } throw err; } } @@ -208,7 +211,9 @@ export class ExecutionEngineHttp implements IExecutionEngine { // If there are errors by EL like connection refused, internal error, they need to be // treated separate from being INVALID. For now, just pass the error upstream. .catch((e: Error): EngineApiRpcReturnTypes[typeof method] => { - this.updateEngineState(getExecutionEngineState({payloadError: e})); + if (!isErrorAborted(e) && !isQueueErrorAborted(e)) { + this.updateEngineState(getExecutionEngineState({payloadError: e})); + } if (e instanceof HttpRpcError || e instanceof ErrorJsonRpcResponse) { return {status: ExecutePayloadStatus.ELERROR, latestValidHash: null, validationError: e.message}; } else { @@ -306,7 +311,9 @@ export class ExecutionEngineHttp implements IExecutionEngine { // If there are errors by EL like connection refused, internal error, they need to be // treated separate from being INVALID. For now, just pass the error upstream. .catch((e: Error): EngineApiRpcReturnTypes[typeof method] => { - this.updateEngineState(getExecutionEngineState({payloadError: e})); + if (!isErrorAborted(e) && !isQueueErrorAborted(e)) { + this.updateEngineState(getExecutionEngineState({payloadError: e})); + } throw e; }); From 4df3774b01cc74e4a10bcab649f9366bf567c5c2 Mon Sep 17 00:00:00 2001 From: g11tech Date: Wed, 19 Jul 2023 00:06:09 +0530 Subject: [PATCH 70/96] feat: add engine fcU v3 for devnet 8 (#5773) --- packages/beacon-node/src/execution/engine/http.ts | 7 ++++++- packages/beacon-node/src/execution/engine/mock.ts | 1 + packages/beacon-node/src/execution/engine/types.ts | 13 +++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index 0a6e18b9ee53..272447eb5bb6 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -294,7 +294,12 @@ export class ExecutionEngineHttp implements IExecutionEngine { ): Promise { // Once on capella, should this need to be permanently switched to v2 when payload attrs // not provided - const method = ForkSeq[fork] >= ForkSeq.capella ? "engine_forkchoiceUpdatedV2" : "engine_forkchoiceUpdatedV1"; + const method = + ForkSeq[fork] >= ForkSeq.deneb + ? "engine_forkchoiceUpdatedV3" + : ForkSeq[fork] >= ForkSeq.capella + ? "engine_forkchoiceUpdatedV2" + : "engine_forkchoiceUpdatedV1"; const payloadAttributesRpc = payloadAttributes ? serializePayloadAttributes(payloadAttributes) : undefined; // If we are just fcUing and not asking execution for payload, retry is not required // and we can move on, as the next fcU will be issued soon on the new slot diff --git a/packages/beacon-node/src/execution/engine/mock.ts b/packages/beacon-node/src/execution/engine/mock.ts index 3e3d3d65f901..eea528d1a50a 100644 --- a/packages/beacon-node/src/execution/engine/mock.ts +++ b/packages/beacon-node/src/execution/engine/mock.ts @@ -90,6 +90,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { engine_newPayloadV3: this.notifyNewPayload.bind(this), engine_forkchoiceUpdatedV1: this.notifyForkchoiceUpdate.bind(this), engine_forkchoiceUpdatedV2: this.notifyForkchoiceUpdate.bind(this), + engine_forkchoiceUpdatedV3: this.notifyForkchoiceUpdate.bind(this), engine_getPayloadV1: this.getPayload.bind(this), engine_getPayloadV2: this.getPayload.bind(this), engine_getPayloadV3: this.getPayload.bind(this), diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index daf66da0106b..14ec66bdcca6 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -41,6 +41,10 @@ export type EngineApiRpcParamTypes = { forkChoiceData: {headBlockHash: DATA; safeBlockHash: DATA; finalizedBlockHash: DATA}, payloadAttributes?: PayloadAttributesRpc, ]; + engine_forkchoiceUpdatedV3: [ + forkChoiceData: {headBlockHash: DATA; safeBlockHash: DATA; finalizedBlockHash: DATA}, + payloadAttributes?: PayloadAttributesRpc, + ]; /** * 1. payloadId: QUANTITY, 64 Bits - Identifier of the payload building process */ @@ -82,6 +86,10 @@ export type EngineApiRpcReturnTypes = { payloadStatus: PayloadStatus; payloadId: QUANTITY | null; }; + engine_forkchoiceUpdatedV3: { + payloadStatus: PayloadStatus; + payloadId: QUANTITY | null; + }; /** * payloadId | Error: QUANTITY, 64 Bits - Identifier of the payload building process */ @@ -123,6 +131,7 @@ export type ExecutionPayloadRpc = { withdrawals?: WithdrawalRpc[]; // Capella hardfork dataGasUsed?: QUANTITY; // DENEB excessDataGas?: QUANTITY; // DENEB + parentBeaconBlockRoot?: QUANTITY; // DENEB }; export type WithdrawalRpc = { @@ -142,6 +151,8 @@ export type PayloadAttributesRpc = { /** DATA, 20 Bytes - suggested value for the coinbase field of the new payload */ suggestedFeeRecipient: DATA; withdrawals?: WithdrawalRpc[]; + /** DATA, 32 Bytes - value for the parentBeaconBlockRoot to be used for building block */ + parentBeaconBlockRoot?: DATA; }; export interface BlobsBundleRpc { @@ -266,6 +277,7 @@ export function serializePayloadAttributes(data: PayloadAttributes): PayloadAttr prevRandao: bytesToData(data.prevRandao), suggestedFeeRecipient: data.suggestedFeeRecipient, withdrawals: data.withdrawals?.map(serializeWithdrawal), + parentBeaconBlockRoot: data.parentBeaconBlockRoot ? bytesToData(data.parentBeaconBlockRoot) : undefined, }; } @@ -281,6 +293,7 @@ export function deserializePayloadAttributes(data: PayloadAttributesRpc): Payloa // avoid any conversions suggestedFeeRecipient: data.suggestedFeeRecipient, withdrawals: data.withdrawals?.map((withdrawal) => deserializeWithdrawal(withdrawal)), + parentBeaconBlockRoot: data.parentBeaconBlockRoot ? dataToBytes(data.parentBeaconBlockRoot, 32) : undefined, }; } From 29314c9e4515662fd882b9a47b8aa0c95b0ff88c Mon Sep 17 00:00:00 2001 From: g11tech Date: Wed, 19 Jul 2023 18:11:24 +0530 Subject: [PATCH 71/96] feat: extend attestation validity till end of next epoch - eip7045 (#5731) * feat: extend attestation validity till end of next epoch - eip7045 upgrade spec version Review PR update p2p validations fix tests * fix tests --- .../src/api/impl/beacon/pool/index.ts | 3 +- .../src/api/impl/validator/index.ts | 3 +- .../src/chain/errors/attestationError.ts | 2 +- .../src/chain/validation/aggregateAndProof.ts | 13 +++- .../src/chain/validation/attestation.ts | 69 ++++++++++++++----- .../src/network/processor/gossipHandlers.ts | 8 ++- .../validation/aggregateAndProof.test.ts | 6 +- .../perf/chain/validation/attestation.test.ts | 6 +- .../test/spec/specTestVersioning.ts | 2 +- .../validation/aggregateAndProof.test.ts | 6 +- .../unit/chain/validation/attestation.test.ts | 12 +++- .../src/block/processAttestationPhase0.ts | 26 +++++-- .../src/block/processAttestations.ts | 2 +- .../src/block/processAttestationsAltair.ts | 17 +++-- .../src/slot/upgradeStateToAltair.ts | 2 + .../perf/block/processAttestation.test.ts | 7 +- 16 files changed, 137 insertions(+), 47 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index f1973fef615e..170c358fe417 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -52,8 +52,9 @@ export function getBeaconPoolApi({ await Promise.all( attestations.map(async (attestation, i) => { try { + const fork = chain.config.getForkName(chain.clock.currentSlot); // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - const validateFn = () => validateApiAttestation(chain, {attestation, serializedData: null}); + const validateFn = () => validateApiAttestation(fork, chain, {attestation, serializedData: null}); const {slot, beaconBlockRoot} = attestation.data; // when a validator is configured with multiple beacon node urls, this attestation data may come from another beacon node // and the block hasn't been in our forkchoice since we haven't seen / processing that block diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 242b6cccce9e..bc0b25a953d6 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -556,13 +556,14 @@ export function getValidatorApi({ const seenTimestampSec = Date.now() / 1000; const errors: Error[] = []; + const fork = chain.config.getForkName(chain.clock.currentSlot); await Promise.all( signedAggregateAndProofs.map(async (signedAggregateAndProof, i) => { try { // TODO: Validate in batch // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - const validateFn = () => validateApiAggregateAndProof(chain, signedAggregateAndProof); + const validateFn = () => validateApiAggregateAndProof(fork, chain, signedAggregateAndProof); const {slot, beaconBlockRoot} = signedAggregateAndProof.message.aggregate.data; // when a validator is configured with multiple beacon node urls, this attestation may come from another beacon node // and the block hasn't been in our forkchoice since we haven't seen / processing that block diff --git a/packages/beacon-node/src/chain/errors/attestationError.ts b/packages/beacon-node/src/chain/errors/attestationError.ts index fa380d048c7e..a93f5b42e439 100644 --- a/packages/beacon-node/src/chain/errors/attestationError.ts +++ b/packages/beacon-node/src/chain/errors/attestationError.ts @@ -154,7 +154,7 @@ export type AttestationErrorType = | {code: AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET} | {code: AttestationErrorCode.PRIOR_ATTESTATION_KNOWN; validatorIndex: ValidatorIndex; epoch: Epoch} | {code: AttestationErrorCode.FUTURE_EPOCH; attestationEpoch: Epoch; currentEpoch: Epoch} - | {code: AttestationErrorCode.PAST_EPOCH; attestationEpoch: Epoch; currentEpoch: Epoch} + | {code: AttestationErrorCode.PAST_EPOCH; attestationEpoch: Epoch; previousEpoch: Epoch} | {code: AttestationErrorCode.ATTESTS_TO_FUTURE_BLOCK; block: Slot; attestation: Slot} | {code: AttestationErrorCode.INVALID_SUBNET_ID; received: number; expected: number} | {code: AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS} diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index aa60140e2d9a..0cd96a8278ec 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -1,4 +1,5 @@ import {toHexString} from "@chainsafe/ssz"; +import {ForkName} from "@lodestar/params"; import {phase0, RootHex, ssz, ValidatorIndex} from "@lodestar/types"; import { computeEpochAtSlot, @@ -26,23 +27,29 @@ export type AggregateAndProofValidationResult = { }; export async function validateApiAggregateAndProof( + fork: ForkName, chain: IBeaconChain, signedAggregateAndProof: phase0.SignedAggregateAndProof ): Promise { const skipValidationKnownAttesters = true; const prioritizeBls = true; - return validateAggregateAndProof(chain, signedAggregateAndProof, null, {skipValidationKnownAttesters, prioritizeBls}); + return validateAggregateAndProof(fork, chain, signedAggregateAndProof, null, { + skipValidationKnownAttesters, + prioritizeBls, + }); } export async function validateGossipAggregateAndProof( + fork: ForkName, chain: IBeaconChain, signedAggregateAndProof: phase0.SignedAggregateAndProof, serializedData: Uint8Array ): Promise { - return validateAggregateAndProof(chain, signedAggregateAndProof, serializedData); + return validateAggregateAndProof(fork, chain, signedAggregateAndProof, serializedData); } async function validateAggregateAndProof( + fork: ForkName, chain: IBeaconChain, signedAggregateAndProof: phase0.SignedAggregateAndProof, serializedData: Uint8Array | null = null, @@ -87,7 +94,7 @@ async function validateAggregateAndProof( // [IGNORE] aggregate.data.slot is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) // -- i.e. aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot // (a client MAY queue future aggregates for processing at the appropriate slot). - verifyPropagationSlotRange(chain, attSlot); + verifyPropagationSlotRange(fork, chain, attSlot); } // [IGNORE] The aggregate is the first valid aggregate received for the aggregator with diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 3a17b3d4a0aa..bea4d060e48a 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -1,7 +1,7 @@ import {toHexString} from "@chainsafe/ssz"; import {phase0, Epoch, Root, Slot, RootHex, ssz} from "@lodestar/types"; import {ProtoBlock} from "@lodestar/fork-choice"; -import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, ForkName, ForkSeq} from "@lodestar/params"; import { computeEpochAtSlot, CachedBeaconStateAllForks, @@ -56,12 +56,13 @@ const SHUFFLING_LOOK_AHEAD_EPOCHS = 1; * - do not prioritize bls signature set */ export async function validateGossipAttestation( + fork: ForkName, chain: IBeaconChain, attestationOrBytes: GossipAttestation, /** Optional, to allow verifying attestations through API with unknown subnet */ subnet: number ): Promise { - return validateAttestation(chain, attestationOrBytes, subnet); + return validateAttestation(fork, chain, attestationOrBytes, subnet); } /** @@ -71,11 +72,12 @@ export async function validateGossipAttestation( * - prioritize bls signature set */ export async function validateApiAttestation( + fork: ForkName, chain: IBeaconChain, attestationOrBytes: ApiAttestation ): Promise { const prioritizeBls = true; - return validateAttestation(chain, attestationOrBytes, null, prioritizeBls); + return validateAttestation(fork, chain, attestationOrBytes, null, prioritizeBls); } /** @@ -83,6 +85,7 @@ export async function validateApiAttestation( * This is to avoid deserializing similar attestation multiple times which could help the gc */ async function validateAttestation( + fork: ForkName, chain: IBeaconChain, attestationOrBytes: AttestationOrBytes, /** Optional, to allow verifying attestations through API with unknown subnet */ @@ -146,7 +149,7 @@ async function validateAttestation( // [IGNORE] attestation.data.slot is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (within a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) // -- i.e. attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot // (a client MAY queue future attestations for processing at the appropriate slot). - verifyPropagationSlotRange(chain, attestationOrCache.attestation.data.slot); + verifyPropagationSlotRange(fork, chain, attestationOrCache.attestation.data.slot); } // [REJECT] The attestation is unaggregated -- that is, it has exactly one participating validator @@ -343,22 +346,9 @@ async function validateAttestation( * Accounts for `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. * Note: We do not queue future attestations for later processing */ -export function verifyPropagationSlotRange(chain: IBeaconChain, attestationSlot: Slot): void { +export function verifyPropagationSlotRange(fork: ForkName, chain: IBeaconChain, attestationSlot: Slot): void { // slot with future tolerance of MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC const latestPermissibleSlot = chain.clock.slotWithFutureTolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC); - const earliestPermissibleSlot = Math.max( - // slot with past tolerance of MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC - // ATTESTATION_PROPAGATION_SLOT_RANGE = SLOTS_PER_EPOCH - chain.clock.slotWithPastTolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC) - SLOTS_PER_EPOCH, - 0 - ); - if (attestationSlot < earliestPermissibleSlot) { - throw new AttestationError(GossipAction.IGNORE, { - code: AttestationErrorCode.PAST_SLOT, - earliestPermissibleSlot, - attestationSlot, - }); - } if (attestationSlot > latestPermissibleSlot) { throw new AttestationError(GossipAction.IGNORE, { code: AttestationErrorCode.FUTURE_SLOT, @@ -366,6 +356,49 @@ export function verifyPropagationSlotRange(chain: IBeaconChain, attestationSlot: attestationSlot, }); } + + const earliestPermissibleSlot = Math.max( + // slot with past tolerance of MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC + // ATTESTATION_PROPAGATION_SLOT_RANGE = SLOTS_PER_EPOCH + chain.clock.slotWithPastTolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC) - SLOTS_PER_EPOCH, + 0 + ); + + // Post deneb the attestations are valid for current as well as previous epoch + // while pre deneb they are valid for ATTESTATION_PROPAGATION_SLOT_RANGE + // + // see: https://github.com/ethereum/consensus-specs/pull/3360 + if (ForkSeq[fork] < ForkSeq.deneb) { + if (attestationSlot < earliestPermissibleSlot) { + throw new AttestationError(GossipAction.IGNORE, { + code: AttestationErrorCode.PAST_SLOT, + earliestPermissibleSlot, + attestationSlot, + }); + } + } else { + const attestationEpoch = computeEpochAtSlot(attestationSlot); + + // upper bound for current epoch is same as epoch of latestPermissibleSlot + const latestPermissibleCurrentEpoch = computeEpochAtSlot(latestPermissibleSlot); + if (attestationEpoch > latestPermissibleCurrentEpoch) { + throw new AttestationError(GossipAction.IGNORE, { + code: AttestationErrorCode.FUTURE_EPOCH, + currentEpoch: latestPermissibleCurrentEpoch, + attestationEpoch, + }); + } + + // lower bound for previous epoch is same as epoch of earliestPermissibleSlot + const earliestPermissiblePreviousEpoch = computeEpochAtSlot(earliestPermissibleSlot); + if (attestationEpoch < earliestPermissiblePreviousEpoch) { + throw new AttestationError(GossipAction.IGNORE, { + code: AttestationErrorCode.PAST_EPOCH, + previousEpoch: earliestPermissiblePreviousEpoch, + attestationEpoch, + }); + } + } } /** diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index e3692b7fb4a3..58d4a9ea3c60 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -192,9 +192,10 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH [GossipType.beacon_aggregate_and_proof]: async ({serializedData}, topic, _peer, seenTimestampSec) => { let validationResult: AggregateAndProofValidationResult; const signedAggregateAndProof = sszDeserialize(topic, serializedData); + const {fork} = topic; try { - validationResult = await validateGossipAggregateAndProof(chain, signedAggregateAndProof, serializedData); + validationResult = await validateGossipAggregateAndProof(fork, chain, signedAggregateAndProof, serializedData); } catch (e) { if (e instanceof AttestationError && e.action === GossipAction.REJECT) { chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "gossip_reject"); @@ -229,14 +230,17 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH chain.emitter.emit(routes.events.EventType.attestation, signedAggregateAndProof.message.aggregate); }, - [GossipType.beacon_attestation]: async ({serializedData, msgSlot}, {subnet}, _peer, seenTimestampSec) => { + [GossipType.beacon_attestation]: async ({serializedData, msgSlot}, topic, _peer, seenTimestampSec) => { if (msgSlot == undefined) { throw Error("msgSlot is undefined for beacon_attestation topic"); } + const {subnet, fork} = topic; + // do not deserialize gossipSerializedData here, it's done in validateGossipAttestation only if needed let validationResult: AttestationValidationResult; try { validationResult = await validateGossipAttestation( + fork, chain, {attestation: null, serializedData, attSlot: msgSlot}, subnet diff --git a/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts index fe8ade59cef2..3fc7efe9d675 100644 --- a/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts @@ -26,7 +26,8 @@ describe("validate gossip signedAggregateAndProof", () => { chain.seenAggregatedAttestations["aggregateRootsByEpoch"].clear(); }, fn: async () => { - await validateApiAggregateAndProof(chain, agg); + const fork = chain.config.getForkName(stateSlot); + await validateApiAggregateAndProof(fork, chain, agg); }, }); @@ -37,7 +38,8 @@ describe("validate gossip signedAggregateAndProof", () => { chain.seenAggregatedAttestations["aggregateRootsByEpoch"].clear(); }, fn: async () => { - await validateGossipAggregateAndProof(chain, agg, serializedData); + const fork = chain.config.getForkName(stateSlot); + await validateGossipAggregateAndProof(fork, chain, agg, serializedData); }, }); } diff --git a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts index b9664c6f5efd..84b423dd7e62 100644 --- a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts @@ -23,7 +23,8 @@ describe("validate attestation", () => { id: `validate api attestation - ${id}`, beforeEach: () => chain.seenAttesters["validatorIndexesByEpoch"].clear(), fn: async () => { - await validateApiAttestation(chain, {attestation: att, serializedData: null}); + const fork = chain.config.getForkName(stateSlot); + await validateApiAttestation(fork, chain, {attestation: att, serializedData: null}); }, }); @@ -31,7 +32,8 @@ describe("validate attestation", () => { id: `validate gossip attestation - ${id}`, beforeEach: () => chain.seenAttesters["validatorIndexesByEpoch"].clear(), fn: async () => { - await validateGossipAttestation(chain, {attestation: null, serializedData, attSlot: slot}, subnet); + const fork = chain.config.getForkName(stateSlot); + await validateGossipAttestation(fork, chain, {attestation: null, serializedData, attSlot: slot}, subnet); }, }); } diff --git a/packages/beacon-node/test/spec/specTestVersioning.ts b/packages/beacon-node/test/spec/specTestVersioning.ts index 2e967e087c69..37f08e3f9e56 100644 --- a/packages/beacon-node/test/spec/specTestVersioning.ts +++ b/packages/beacon-node/test/spec/specTestVersioning.ts @@ -15,7 +15,7 @@ import {DownloadTestsOptions} from "@lodestar/spec-test-util"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); export const ethereumConsensusSpecsTests: DownloadTestsOptions = { - specVersion: "v1.4.0-alpha.3", + specVersion: "v1.4.0-beta.0", // Target directory is the host package root: 'packages/*/spec-tests' outputDir: path.join(__dirname, "../../spec-tests"), specTestsRepoUrl: "https://github.com/ethereum/consensus-spec-tests", diff --git a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts index 54191ba34846..20300776a2ec 100644 --- a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts @@ -42,7 +42,8 @@ describe("chain / validation / aggregateAndProof", () => { it("Valid", async () => { const {chain, signedAggregateAndProof} = getValidData({}); - await validateApiAggregateAndProof(chain, signedAggregateAndProof); + const fork = chain.config.getForkName(stateSlot); + await validateApiAggregateAndProof(fork, chain, signedAggregateAndProof); }); it("BAD_TARGET_EPOCH", async () => { @@ -188,9 +189,10 @@ describe("chain / validation / aggregateAndProof", () => { signedAggregateAndProof: phase0.SignedAggregateAndProof, errorCode: AttestationErrorCode ): Promise { + const fork = chain.config.getForkName(stateSlot); const serializedData = ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof); await expectRejectedWithLodestarError( - validateGossipAggregateAndProof(chain, signedAggregateAndProof, serializedData), + validateGossipAggregateAndProof(fork, chain, signedAggregateAndProof, serializedData), errorCode ); } diff --git a/packages/beacon-node/test/unit/chain/validation/attestation.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation.test.ts index 6296ecb4c6fc..f28f62abc229 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation.test.ts @@ -52,7 +52,8 @@ describe("chain / validation / attestation", () => { it("Valid", async () => { const {chain, attestation} = getValidData(); - await validateApiAttestation(chain, {attestation, serializedData: null}); + const fork = chain.config.getForkName(stateSlot); + await validateApiAttestation(fork, chain, {attestation, serializedData: null}); }); it("INVALID_SERIALIZED_BYTES_ERROR_CODE", async () => { @@ -276,7 +277,8 @@ describe("chain / validation / attestation", () => { attestationOrBytes: ApiAttestation, errorCode: string ): Promise { - await expectRejectedWithLodestarError(validateApiAttestation(chain, attestationOrBytes), errorCode); + const fork = chain.config.getForkName(stateSlot); + await expectRejectedWithLodestarError(validateApiAttestation(fork, chain, attestationOrBytes), errorCode); } async function expectGossipError( @@ -285,7 +287,11 @@ describe("chain / validation / attestation", () => { subnet: number, errorCode: string ): Promise { - await expectRejectedWithLodestarError(validateGossipAttestation(chain, attestationOrBytes, subnet), errorCode); + const fork = chain.config.getForkName(stateSlot); + await expectRejectedWithLodestarError( + validateGossipAttestation(fork, chain, attestationOrBytes, subnet), + errorCode + ); } }); diff --git a/packages/state-transition/src/block/processAttestationPhase0.ts b/packages/state-transition/src/block/processAttestationPhase0.ts index 6e7ae506dc93..248ba83b4ed2 100644 --- a/packages/state-transition/src/block/processAttestationPhase0.ts +++ b/packages/state-transition/src/block/processAttestationPhase0.ts @@ -1,7 +1,7 @@ import {toHexString} from "@chainsafe/ssz"; -import {phase0, ssz} from "@lodestar/types"; +import {Slot, phase0, ssz} from "@lodestar/types"; -import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH, ForkSeq} from "@lodestar/params"; import {computeEpochAtSlot} from "../util/index.js"; import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../types.js"; import {isValidIndexedAttestation} from "./index.js"; @@ -22,7 +22,7 @@ export function processAttestationPhase0( const slot = state.slot; const data = attestation.data; - validateAttestation(state, attestation); + validateAttestation(ForkSeq.phase0, state, attestation); const pendingAttestation = ssz.phase0.PendingAttestation.toViewDU({ data: data, @@ -56,7 +56,11 @@ export function processAttestationPhase0( } } -export function validateAttestation(state: CachedBeaconStateAllForks, attestation: phase0.Attestation): void { +export function validateAttestation( + fork: ForkSeq, + state: CachedBeaconStateAllForks, + attestation: phase0.Attestation +): void { const {epochCtx} = state; const slot = state.slot; const data = attestation.data; @@ -80,7 +84,9 @@ export function validateAttestation(state: CachedBeaconStateAllForks, attestatio `targetEpoch=${data.target.epoch} computedEpoch=${computedEpoch}` ); } - if (!(data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= slot && slot <= data.slot + SLOTS_PER_EPOCH)) { + + // post deneb, the attestations are valid till end of next epoch + if (!(data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= slot && isTimelyTarget(fork, slot - data.slot))) { throw new Error( "Attestation slot not within inclusion window: " + `slot=${data.slot} window=${data.slot + MIN_ATTESTATION_INCLUSION_DELAY}..${data.slot + SLOTS_PER_EPOCH}` @@ -96,6 +102,16 @@ export function validateAttestation(state: CachedBeaconStateAllForks, attestatio } } +// Modified https://github.com/ethereum/consensus-specs/pull/3360 +export function isTimelyTarget(fork: ForkSeq, inclusionDistance: Slot): boolean { + // post deneb attestation is valid till end of next epoch for target + if (fork >= ForkSeq.deneb) { + return true; + } else { + return inclusionDistance <= SLOTS_PER_EPOCH; + } +} + export function checkpointToStr(checkpoint: phase0.Checkpoint): string { return `${toHexString(checkpoint.root)}:${checkpoint.epoch}`; } diff --git a/packages/state-transition/src/block/processAttestations.ts b/packages/state-transition/src/block/processAttestations.ts index a2976e6491c3..2b132fa22e0b 100644 --- a/packages/state-transition/src/block/processAttestations.ts +++ b/packages/state-transition/src/block/processAttestations.ts @@ -18,6 +18,6 @@ export function processAttestations( processAttestationPhase0(state as CachedBeaconStatePhase0, attestation, verifySignatures); } } else { - processAttestationsAltair(state as CachedBeaconStateAltair, attestations, verifySignatures); + processAttestationsAltair(fork, state as CachedBeaconStateAltair, attestations, verifySignatures); } } diff --git a/packages/state-transition/src/block/processAttestationsAltair.ts b/packages/state-transition/src/block/processAttestationsAltair.ts index a77a6d1de0bd..cabe28b88b27 100644 --- a/packages/state-transition/src/block/processAttestationsAltair.ts +++ b/packages/state-transition/src/block/processAttestationsAltair.ts @@ -13,12 +13,13 @@ import { TIMELY_TARGET_FLAG_INDEX, TIMELY_TARGET_WEIGHT, WEIGHT_DENOMINATOR, + ForkSeq, } from "@lodestar/params"; import {increaseBalance, verifySignatureSet} from "../util/index.js"; import {CachedBeaconStateAltair} from "../types.js"; import {RootCache} from "../util/rootCache.js"; import {getAttestationWithIndicesSignatureSet} from "../signatureSets/indexedAttestation.js"; -import {checkpointToStr, validateAttestation} from "./processAttestationPhase0.js"; +import {checkpointToStr, isTimelyTarget, validateAttestation} from "./processAttestationPhase0.js"; const PROPOSER_REWARD_DOMINATOR = ((WEIGHT_DENOMINATOR - PROPOSER_WEIGHT) * WEIGHT_DENOMINATOR) / PROPOSER_WEIGHT; @@ -29,6 +30,7 @@ const TIMELY_HEAD = 1 << TIMELY_HEAD_FLAG_INDEX; const SLOTS_PER_EPOCH_SQRT = intSqrt(SLOTS_PER_EPOCH); export function processAttestationsAltair( + fork: ForkSeq, state: CachedBeaconStateAltair, attestations: phase0.Attestation[], verifySignature = true @@ -44,7 +46,7 @@ export function processAttestationsAltair( for (const attestation of attestations) { const data = attestation.data; - validateAttestation(state, attestation); + validateAttestation(fork, state, attestation); // Retrieve the validator indices from the attestation participation bitfield const committeeIndices = epochCtx.getBeaconCommittee(data.slot, data.index); @@ -63,7 +65,13 @@ export function processAttestationsAltair( const inCurrentEpoch = data.target.epoch === currentEpoch; const epochParticipation = inCurrentEpoch ? state.currentEpochParticipation : state.previousEpochParticipation; - const flagsAttestation = getAttestationParticipationStatus(data, stateSlot - data.slot, epochCtx.epoch, rootCache); + const flagsAttestation = getAttestationParticipationStatus( + fork, + data, + stateSlot - data.slot, + epochCtx.epoch, + rootCache + ); // For each participant, update their participation // In epoch processing, this participation info is used to calculate balance updates @@ -121,6 +129,7 @@ export function processAttestationsAltair( * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/altair/beacon-chain.md#get_attestation_participation_flag_indices */ export function getAttestationParticipationStatus( + fork: ForkSeq, data: phase0.AttestationData, inclusionDelay: number, currentEpoch: Epoch, @@ -152,7 +161,7 @@ export function getAttestationParticipationStatus( let flags = 0; if (isMatchingSource && inclusionDelay <= SLOTS_PER_EPOCH_SQRT) flags |= TIMELY_SOURCE; - if (isMatchingTarget && inclusionDelay <= SLOTS_PER_EPOCH) flags |= TIMELY_TARGET; + if (isMatchingTarget && isTimelyTarget(fork, inclusionDelay)) flags |= TIMELY_TARGET; if (isMatchingHead && inclusionDelay === MIN_ATTESTATION_INCLUSION_DELAY) flags |= TIMELY_HEAD; return flags; diff --git a/packages/state-transition/src/slot/upgradeStateToAltair.ts b/packages/state-transition/src/slot/upgradeStateToAltair.ts index a7af38c8f068..0afa43930ef0 100644 --- a/packages/state-transition/src/slot/upgradeStateToAltair.ts +++ b/packages/state-transition/src/slot/upgradeStateToAltair.ts @@ -1,5 +1,6 @@ import {CompositeViewDU} from "@chainsafe/ssz"; import {ssz} from "@lodestar/types"; +import {ForkSeq} from "@lodestar/params"; import {CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../types.js"; import {newZeroedArray, RootCache} from "../util/index.js"; import {getNextSyncCommittee} from "../util/syncCommittee.js"; @@ -128,6 +129,7 @@ function translateParticipation( for (const attestation of pendingAttesations.getAllReadonly()) { const data = attestation.data; const attestationFlags = getAttestationParticipationStatus( + ForkSeq.altair, data, attestation.inclusionDelay, epochCtx.epoch, diff --git a/packages/state-transition/test/perf/block/processAttestation.test.ts b/packages/state-transition/test/perf/block/processAttestation.test.ts index d1dea1ff15bd..673b0e17430f 100644 --- a/packages/state-transition/test/perf/block/processAttestation.test.ts +++ b/packages/state-transition/test/perf/block/processAttestation.test.ts @@ -72,7 +72,12 @@ describe("altair processAttestation", () => { return {state: stateCloned, attestations}; }, fn: ({state, attestations}) => { - processAttestationsAltair(state as CachedBeaconStateAltair, attestations, false); + processAttestationsAltair( + state.config.getForkSeq(state.slot), + state as CachedBeaconStateAltair, + attestations, + false + ); state.commit(); // After processAttestations normal case vc 250_000 it has to do 6802 hash64 ops // state.hashTreeRoot(); From fce550cf7d32110791b624a5f752ae54f09350fa Mon Sep 17 00:00:00 2001 From: Phil Ngo <58080811+philknows@users.noreply.github.com> Date: Thu, 20 Jul 2023 00:23:53 -0400 Subject: [PATCH 72/96] chore: automatic meta-bug labels for bug report issues template (#5776) Update bug_report.yaml --- .github/ISSUE_TEMPLATE/bug_report.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index c2f9837e08ba..c5bfaf1b533e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -1,6 +1,7 @@ name: Bug report description: Create a bug report to help us improve title: "[Descriptive title] " +labels: [meta-bug] body: - type: textarea id: describe From 81f7ab5ffa55af3131b7d4647bd9a51b04ad4e4e Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 20 Jul 2023 14:59:22 +0200 Subject: [PATCH 73/96] fix: metrics server listen on localhost by default (#5777) --- packages/beacon-node/src/metrics/options.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/beacon-node/src/metrics/options.ts b/packages/beacon-node/src/metrics/options.ts index 09041c04b440..9a19e033a291 100644 --- a/packages/beacon-node/src/metrics/options.ts +++ b/packages/beacon-node/src/metrics/options.ts @@ -18,4 +18,5 @@ export type MetricsOptions = HttpMetricsServerOpts & { export const defaultMetricsOptions: MetricsOptions = { enabled: false, port: 8008, + address: "127.0.0.1", }; From 2c6ef6bed943a1491b384b862a57ae1eeba44575 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 20 Jul 2023 15:01:02 +0200 Subject: [PATCH 74/96] refactor: align metrics and REST API server logs (#5778) --- packages/beacon-node/src/metrics/server/http.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/beacon-node/src/metrics/server/http.ts b/packages/beacon-node/src/metrics/server/http.ts index 5a7943750007..ba17b743d112 100644 --- a/packages/beacon-node/src/metrics/server/http.ts +++ b/packages/beacon-node/src/metrics/server/http.ts @@ -1,4 +1,5 @@ import http from "node:http"; +import {AddressInfo} from "node:net"; import {Registry} from "prom-client"; import {Logger} from "@lodestar/utils"; import {wrapError} from "../../util/wrapError.js"; @@ -72,12 +73,17 @@ export async function getHttpMetricsServer( const activeSockets = new HttpActiveSocketsTracker(server, socketsMetrics); - const {port, address} = opts; - logger.info("Starting metrics HTTP server", {port, address: address ?? "127.0.0.1"}); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(port, address, resolve); + server.once("error", (err) => { + logger.error("Error starting metrics HTTP server", opts, err); + reject(err); + }); + server.listen(opts.port, opts.address, () => { + const {port, address: host, family} = server.address() as AddressInfo; + const address = `http://${family === "IPv6" ? `[${host}]` : host}:${port}`; + logger.info("Started metrics HTTP server", {address}); + resolve(); + }); }); return { From 4e6408ac7a44d47fa63edee7444cb2132dc940db Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 20 Jul 2023 15:26:30 +0200 Subject: [PATCH 75/96] refactor: update execution client state change logs (#5771) * Update execution client state change message * Use warn instead of info if execution client is syncing --- packages/beacon-node/src/execution/engine/http.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index 272447eb5bb6..ea89506c2fc4 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -440,19 +440,19 @@ export class ExecutionEngineHttp implements IExecutionEngine { switch (newState) { case ExecutionEngineState.ONLINE: - this.logger.info("ExecutionEngine became online"); + this.logger.info("Execution client became online"); break; case ExecutionEngineState.OFFLINE: - this.logger.error("ExecutionEngine went offline"); + this.logger.error("Execution client went offline"); break; case ExecutionEngineState.SYNCED: - this.logger.info("ExecutionEngine is synced"); + this.logger.info("Execution client is synced"); break; case ExecutionEngineState.SYNCING: - this.logger.info("ExecutionEngine is syncing"); + this.logger.warn("Execution client is syncing"); break; case ExecutionEngineState.AUTH_FAILED: - this.logger.error("ExecutionEngine authentication failed"); + this.logger.error("Execution client authentication failed"); break; } From 6a5225f89ff3f207e12fde53155c79779c48f975 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 20 Jul 2023 11:59:12 -0400 Subject: [PATCH 76/96] chore: bump test-utils to 1.9.2 (#5781) * chore: bump test-utils to 1.9.2 * Bump @lodestar/utils to 1.9.2 --------- Co-authored-by: Nico Flaig --- packages/cli/package.json | 2 +- packages/prover/package.json | 2 +- packages/test-utils/package.json | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 3d8f28cd2e96..29cfbbfba511 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -100,6 +100,6 @@ "@types/inquirer": "^9.0.3", "@types/lodash": "^4.14.192", "@types/yargs": "^17.0.24", - "@lodestar/test-utils": "^1.9.1" + "@lodestar/test-utils": "^1.9.2" } } diff --git a/packages/prover/package.json b/packages/prover/package.json index 38d03838d1c8..171d0853da66 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -82,7 +82,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.9.1", + "@lodestar/test-utils": "^1.9.2", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 061e1313c445..78caa24cc6e4 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.9.1", + "version": "1.9.2", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -61,11 +61,11 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.9.1", + "@lodestar/utils": "^1.9.2", "axios": "^1.3.4", "chai": "^4.3.7", "mocha": "^10.2.0", - "sinon": "^15.0.3" + "sinon": "^15.0.3" }, "devDependencies": { "@types/mocha": "^10.0.1", From 50551d35670f7a63b37203ae9681b1650fdbf1ac Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Fri, 21 Jul 2023 08:22:28 +0700 Subject: [PATCH 77/96] feat: add verifySignatureSetsSameMessage BLS api (#5747) * Add verifySignatureSetsSameMessage * chore: finish the implementation * fix: handle cannot aggregate pubkeys and signatures in same message * fix: jobItemSameMessageToMultiSet considering priority * fix: handle malformed signatures * fix: multithread e2e test * chore: remove unused metric * fix: correct verifySignatureSetsSameMessage opts * chore: refactor to retryJobItemSameMessage method * Review PR --------- Co-authored-by: dapplion <35266934+dapplion@users.noreply.github.com> --- .../beacon-node/src/chain/bls/interface.ts | 13 + .../beacon-node/src/chain/bls/maybeBatch.ts | 47 +-- .../src/chain/bls/multithread/index.ts | 270 ++++++++++++------ .../src/chain/bls/multithread/jobItem.ts | 115 ++++++++ .../src/chain/bls/multithread/types.ts | 3 +- .../beacon-node/src/chain/bls/singleThread.ts | 47 ++- .../src/metrics/metrics/lodestar.ts | 19 +- .../test/e2e/chain/bls/multithread.test.ts | 85 ++++-- .../test/unit/chain/bls/bls.test.ts | 89 ++++++ .../test/unit/chain/validation/block.test.ts | 7 +- packages/beacon-node/test/utils/mocks/bls.ts | 5 + 11 files changed, 563 insertions(+), 137 deletions(-) create mode 100644 packages/beacon-node/src/chain/bls/multithread/jobItem.ts create mode 100644 packages/beacon-node/test/unit/chain/bls/bls.test.ts diff --git a/packages/beacon-node/src/chain/bls/interface.ts b/packages/beacon-node/src/chain/bls/interface.ts index 0b704e91e201..e9c98ba1920e 100644 --- a/packages/beacon-node/src/chain/bls/interface.ts +++ b/packages/beacon-node/src/chain/bls/interface.ts @@ -1,3 +1,4 @@ +import {PublicKey} from "@chainsafe/bls/types"; import {ISignatureSet} from "@lodestar/state-transition"; export type VerifySignatureOpts = { @@ -45,6 +46,18 @@ export interface IBlsVerifier { */ verifySignatureSets(sets: ISignatureSet[], opts?: VerifySignatureOpts): Promise; + /** + * Similar to verifySignatureSets but: + * - all signatures have the same message + * - return an array of boolean, each element indicates whether the corresponding signature set is valid + * - only support `batchable` option + */ + verifySignatureSetsSameMessage( + sets: {publicKey: PublicKey; signature: Uint8Array}[], + messsage: Uint8Array, + opts?: Omit + ): Promise; + /** For multithread pool awaits terminating all workers */ close(): Promise; diff --git a/packages/beacon-node/src/chain/bls/maybeBatch.ts b/packages/beacon-node/src/chain/bls/maybeBatch.ts index 57a63115b8cd..619ddf4d72ec 100644 --- a/packages/beacon-node/src/chain/bls/maybeBatch.ts +++ b/packages/beacon-node/src/chain/bls/maybeBatch.ts @@ -14,26 +14,33 @@ export type SignatureSetDeserialized = { * Abstracted in a separate file to be consumed by the threaded pool and the main thread implementation. */ export function verifySignatureSetsMaybeBatch(sets: SignatureSetDeserialized[]): boolean { - if (sets.length >= MIN_SET_COUNT_TO_BATCH) { - return bls.Signature.verifyMultipleSignatures( - sets.map((s) => ({ - publicKey: s.publicKey, - message: s.message, - // true = validate signature - signature: bls.Signature.fromBytes(s.signature, CoordType.affine, true), - })) - ); - } + try { + if (sets.length >= MIN_SET_COUNT_TO_BATCH) { + return bls.Signature.verifyMultipleSignatures( + sets.map((s) => ({ + publicKey: s.publicKey, + message: s.message, + // true = validate signature + signature: bls.Signature.fromBytes(s.signature, CoordType.affine, true), + })) + ); + } - // .every on an empty array returns true - if (sets.length === 0) { - throw Error("Empty signature set"); - } + // .every on an empty array returns true + if (sets.length === 0) { + throw Error("Empty signature set"); + } - // If too few signature sets verify them without batching - return sets.every((set) => { - // true = validate signature - const sig = bls.Signature.fromBytes(set.signature, CoordType.affine, true); - return sig.verify(set.publicKey, set.message); - }); + // If too few signature sets verify them without batching + return sets.every((set) => { + // true = validate signature + const sig = bls.Signature.fromBytes(set.signature, CoordType.affine, true); + return sig.verify(set.publicKey, set.message); + }); + } catch (_) { + // A signature could be malformed, in that case fromBytes throws error + // blst-ts `verifyMultipleSignatures` is also a fallible operation if mul_n_aggregate fails + // see https://github.com/ChainSafe/blst-ts/blob/b1ba6333f664b08e5c50b2b0d18c4f079203962b/src/lib.ts#L291 + return false; + } } diff --git a/packages/beacon-node/src/chain/bls/multithread/index.ts b/packages/beacon-node/src/chain/bls/multithread/index.ts index 7c3141454ce2..81990a97a019 100644 --- a/packages/beacon-node/src/chain/bls/multithread/index.ts +++ b/packages/beacon-node/src/chain/bls/multithread/index.ts @@ -7,7 +7,7 @@ import {spawn, Worker} from "@chainsafe/threads"; // eslint-disable-next-line self = undefined; import bls from "@chainsafe/bls"; -import {Implementation, PointFormat} from "@chainsafe/bls/types"; +import {Implementation, PointFormat, PublicKey} from "@chainsafe/bls/types"; import {Logger} from "@lodestar/utils"; import {ISignatureSet} from "@lodestar/state-transition"; import {QueueError, QueueErrorCode} from "../../../util/queue/index.js"; @@ -16,9 +16,17 @@ import {IBlsVerifier, VerifySignatureOpts} from "../interface.js"; import {getAggregatedPubkey, getAggregatedPubkeysCount} from "../utils.js"; import {verifySignatureSetsMaybeBatch} from "../maybeBatch.js"; import {LinkedList} from "../../../util/array.js"; -import {BlsWorkReq, BlsWorkResult, WorkerData, WorkResultCode} from "./types.js"; +import {BlsWorkReq, BlsWorkResult, WorkerData, WorkResultCode, WorkResultError} from "./types.js"; import {chunkifyMaximizeChunkSize} from "./utils.js"; import {defaultPoolSize} from "./poolSize.js"; +import { + JobQueueItem, + JobQueueItemSameMessage, + JobQueueItemType, + jobItemSameMessageToMultiSet, + jobItemSigSets, + jobItemWorkReq, +} from "./jobItem.js"; export type BlsMultiThreadWorkerPoolModules = { logger: Logger; @@ -66,13 +74,6 @@ type WorkerApi = { verifyManySignatureSets(workReqArr: BlsWorkReq[]): Promise; }; -type JobQueueItem = { - resolve: (result: R | PromiseLike) => void; - reject: (error?: Error) => void; - addedTimeMs: number; - workReq: BlsWorkReq; -}; - enum WorkerStatusCode { notInitialized, initializing, @@ -179,15 +180,18 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { // Split large array of sets into smaller. // Very helpful when syncing finalized, sync may submit +1000 sets so chunkify allows to distribute to many workers const results = await Promise.all( - chunkifyMaximizeChunkSize(sets, MAX_SIGNATURE_SETS_PER_JOB).map((setsWorker) => - this.queueBlsWork({ - opts, - sets: setsWorker.map((s) => ({ - publicKey: getAggregatedPubkey(s).toBytes(this.format), - message: s.signingRoot, - signature: s.signature, - })), - }) + chunkifyMaximizeChunkSize(sets, MAX_SIGNATURE_SETS_PER_JOB).map( + (setsChunk) => + new Promise((resolve, reject) => { + return this.queueBlsWork({ + type: JobQueueItemType.default, + resolve, + reject, + addedTimeMs: Date.now(), + opts, + sets: setsChunk, + }); + }) ) ); @@ -199,6 +203,35 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { return results.every((isValid) => isValid === true); } + /** + * Verify signature sets of the same message, only supports worker verification. + */ + async verifySignatureSetsSameMessage( + sets: {publicKey: PublicKey; signature: Uint8Array}[], + message: Uint8Array, + opts: Omit = {} + ): Promise { + // chunkify so that it reduce the risk of retrying when there is at least one invalid signature + const results = await Promise.all( + chunkifyMaximizeChunkSize(sets, MAX_SIGNATURE_SETS_PER_JOB).map( + (setsChunk) => + new Promise((resolve, reject) => { + this.queueBlsWork({ + type: JobQueueItemType.sameMessage, + resolve, + reject, + addedTimeMs: Date.now(), + opts, + sets: setsChunk, + message, + }); + }) + ) + ); + + return results.flat(); + } + async close(): Promise { if (this.bufferedJobs) { clearTimeout(this.bufferedJobs.timeout); @@ -261,7 +294,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { /** * Register BLS work to be done eventually in a worker */ - private async queueBlsWork(workReq: BlsWorkReq): Promise { + private queueBlsWork(job: JobQueueItem): void { if (this.closed) { throw new QueueError({code: QueueErrorCode.QUEUE_ABORTED}); } @@ -275,46 +308,41 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { this.workers[0].status.code === WorkerStatusCode.initializationError && this.workers.every((worker) => worker.status.code === WorkerStatusCode.initializationError) ) { - throw this.workers[0].status.error; + return job.reject(this.workers[0].status.error); } - return new Promise((resolve, reject) => { - const job = {resolve, reject, addedTimeMs: Date.now(), workReq}; - - // below is for non-priority jobs - // Append batchable sets to `bufferedJobs`, starting a timeout to push them into `jobs`. - // Do not call `runJob()`, it is called from `runBufferedJobs()` - if (workReq.opts.batchable) { - if (!this.bufferedJobs) { - this.bufferedJobs = { - jobs: new LinkedList(), - prioritizedJobs: new LinkedList(), - sigCount: 0, - firstPush: Date.now(), - timeout: setTimeout(this.runBufferedJobs, MAX_BUFFER_WAIT_MS), - }; - } - const jobs = workReq.opts.priority ? this.bufferedJobs.prioritizedJobs : this.bufferedJobs.jobs; - jobs.push(job); - this.bufferedJobs.sigCount += job.workReq.sets.length; - if (this.bufferedJobs.sigCount > MAX_BUFFERED_SIGS) { - clearTimeout(this.bufferedJobs.timeout); - this.runBufferedJobs(); - } + // Append batchable sets to `bufferedJobs`, starting a timeout to push them into `jobs`. + // Do not call `runJob()`, it is called from `runBufferedJobs()` + if (job.opts.batchable) { + if (!this.bufferedJobs) { + this.bufferedJobs = { + jobs: new LinkedList(), + prioritizedJobs: new LinkedList(), + sigCount: 0, + firstPush: Date.now(), + timeout: setTimeout(this.runBufferedJobs, MAX_BUFFER_WAIT_MS), + }; } + const jobs = job.opts.priority ? this.bufferedJobs.prioritizedJobs : this.bufferedJobs.jobs; + jobs.push(job); + this.bufferedJobs.sigCount += jobItemSigSets(job); + if (this.bufferedJobs.sigCount > MAX_BUFFERED_SIGS) { + clearTimeout(this.bufferedJobs.timeout); + this.runBufferedJobs(); + } + } - // Push job and schedule to call `runJob` in the next macro event loop cycle. - // This is useful to allow batching job submitted from a synchronous for loop, - // and to prevent large stacks since runJob may be called recursively. - else { - if (workReq.opts.priority) { - this.jobs.unshift(job); - } else { - this.jobs.push(job); - } - setTimeout(this.runJob, 0); + // Push job and schedule to call `runJob` in the next macro event loop cycle. + // This is useful to allow batching job submitted from a synchronous for loop, + // and to prevent large stacks since runJob may be called recursively. + else { + if (job.opts.priority) { + this.jobs.unshift(job); + } else { + this.jobs.push(job); } - }); + setTimeout(this.runJob, 0); + } } /** @@ -332,8 +360,8 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { } // Prepare work package - const jobs = this.prepareWork(); - if (jobs.length === 0) { + const jobsInput = this.prepareWork(); + if (jobsInput.length === 0) { return; } @@ -346,22 +374,63 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { this.workersBusy++; try { - let startedSigSets = 0; - for (const job of jobs) { + let startedJobsDefault = 0; + let startedJobsSameMessage = 0; + let startedSetsDefault = 0; + let startedSetsSameMessage = 0; + const workReqs: BlsWorkReq[] = []; + const jobsStarted: JobQueueItem[] = []; + + for (const job of jobsInput) { this.metrics?.blsThreadPool.jobWaitTime.observe((Date.now() - job.addedTimeMs) / 1000); - startedSigSets += job.workReq.sets.length; + + let workReq: BlsWorkReq; + try { + // Note: This can throw, must be handled per-job. + // Pubkey and signature aggregation is defered here + workReq = jobItemWorkReq(job, this.format); + } catch (e) { + this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type}); + + switch (job.type) { + case JobQueueItemType.default: + job.reject(e as Error); + break; + + case JobQueueItemType.sameMessage: + // there could be an invalid pubkey/signature, retry each individually + this.retryJobItemSameMessage(job); + break; + } + + continue; + } + // Re-push all jobs with matching workReq for easier accounting of results + workReqs.push(workReq); + jobsStarted.push(job); + + if (job.type === JobQueueItemType.sameMessage) { + startedJobsSameMessage += 1; + startedSetsSameMessage += job.sets.length; + } else { + startedJobsDefault += 1; + startedSetsDefault += job.sets.length; + } } + const startedSigSets = startedSetsDefault + startedSetsSameMessage; this.metrics?.blsThreadPool.totalJobsGroupsStarted.inc(1); - this.metrics?.blsThreadPool.totalJobsStarted.inc(jobs.length); - this.metrics?.blsThreadPool.totalSigSetsStarted.inc(startedSigSets); + this.metrics?.blsThreadPool.totalJobsStarted.inc({type: JobQueueItemType.default}, startedJobsDefault); + this.metrics?.blsThreadPool.totalJobsStarted.inc({type: JobQueueItemType.sameMessage}, startedJobsSameMessage); + this.metrics?.blsThreadPool.totalSigSetsStarted.inc({type: JobQueueItemType.default}, startedSetsDefault); + this.metrics?.blsThreadPool.totalSigSetsStarted.inc({type: JobQueueItemType.sameMessage}, startedSetsSameMessage); // Send work package to the worker // If the job, metrics or any code below throws: the job will reject never going stale. // Only downside is the the job promise may be resolved twice, but that's not an issue const jobStartNs = process.hrtime.bigint(); - const workResult = await workerApi.verifyManySignatureSets(jobs.map((job) => job.workReq)); + const workResult = await workerApi.verifyManySignatureSets(workReqs); const jobEndNs = process.hrtime.bigint(); const {workerId, batchRetries, batchSigsSuccess, workerStartNs, workerEndNs, results} = workResult; @@ -369,21 +438,39 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { let errorCount = 0; // Un-wrap work package - for (let i = 0; i < jobs.length; i++) { - const job = jobs[i]; + for (let i = 0; i < jobsStarted.length; i++) { + const job = jobsStarted[i]; const jobResult = results[i]; - const sigSetCount = job.workReq.sets.length; - if (!jobResult) { - job.reject(Error(`No jobResult for index ${i}`)); - errorCount += sigSetCount; - } else if (jobResult.code === WorkResultCode.success) { - job.resolve(jobResult.result); - successCount += sigSetCount; - } else { - const workerError = Error(jobResult.error.message); - if (jobResult.error.stack) workerError.stack = jobResult.error.stack; - job.reject(workerError); - errorCount += sigSetCount; + const sigSetCount = jobItemSigSets(job); + + // TODO: enable exhaustive switch case checks lint rule + switch (job.type) { + case JobQueueItemType.default: + if (!jobResult || jobResult.code !== WorkResultCode.success) { + job.reject(getJobResultError(jobResult, i)); + errorCount += sigSetCount; + } else { + job.resolve(jobResult.result); + successCount += sigSetCount; + } + break; + + // handle result of the verification of aggregated signature against aggregated pubkeys + case JobQueueItemType.sameMessage: + if (!jobResult || jobResult.code !== WorkResultCode.success) { + job.reject(getJobResultError(jobResult, i)); + errorCount += 1; + } else { + if (jobResult.result) { + // All are valid, most of the time it goes here + job.resolve(job.sets.map(() => true)); + } else { + // Retry each individually + this.retryJobItemSameMessage(job); + } + successCount += 1; + } + break; } } @@ -401,9 +488,11 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { this.metrics?.blsThreadPool.batchSigsSuccess.inc(batchSigsSuccess); } catch (e) { // Worker communications should never reject - if (!this.closed) this.logger.error("BlsMultiThreadWorkerPool error", {}, e as Error); + if (!this.closed) { + this.logger.error("BlsMultiThreadWorkerPool error", {}, e as Error); + } // Reject all - for (const job of jobs) { + for (const job of jobsInput) { job.reject(e as Error); } } @@ -418,8 +507,8 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { /** * Grab pending work up to a max number of signatures */ - private prepareWork(): JobQueueItem[] { - const jobs: JobQueueItem[] = []; + private prepareWork(): JobQueueItem[] { + const jobs: JobQueueItem[] = []; let totalSigs = 0; while (totalSigs < MAX_SIGNATURE_SETS_PER_JOB) { @@ -429,7 +518,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { } jobs.push(job); - totalSigs += job.workReq.sets.length; + totalSigs += jobItemSigSets(job); } return jobs; @@ -451,6 +540,19 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { } }; + private retryJobItemSameMessage(job: JobQueueItemSameMessage): void { + // Create new jobs for each pubkey set, and Promise.all all the results + for (const j of jobItemSameMessageToMultiSet(job)) { + if (j.opts.priority) { + this.jobs.unshift(j); + } else { + this.jobs.push(j); + } + } + this.metrics?.blsThreadPool.sameMessageRetryJobs.inc(1); + this.metrics?.blsThreadPool.sameMessageRetrySets.inc(job.sets.length); + } + /** For testing */ private async waitTillInitialized(): Promise { await Promise.all( @@ -462,3 +564,9 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { ); } } + +function getJobResultError(jobResult: WorkResultError | null, i: number): Error { + const workerError = jobResult ? Error(jobResult.error.message) : Error(`No jobResult for index ${i}`); + if (jobResult?.error?.stack) workerError.stack = jobResult.error.stack; + return workerError; +} diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts new file mode 100644 index 000000000000..5e64ba334360 --- /dev/null +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -0,0 +1,115 @@ +import bls from "@chainsafe/bls"; +import {CoordType, PointFormat, PublicKey} from "@chainsafe/bls/types"; +import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; +import {VerifySignatureOpts} from "../interface.js"; +import {getAggregatedPubkey} from "../utils.js"; +import {LinkedList} from "../../../util/array.js"; +import {BlsWorkReq} from "./types.js"; + +export type JobQueueItem = JobQueueItemDefault | JobQueueItemSameMessage; + +export type JobQueueItemDefault = { + type: JobQueueItemType.default; + resolve: (result: boolean) => void; + reject: (error?: Error) => void; + addedTimeMs: number; + opts: VerifySignatureOpts; + sets: ISignatureSet[]; +}; + +export type JobQueueItemSameMessage = { + type: JobQueueItemType.sameMessage; + resolve: (result: boolean[]) => void; + reject: (error?: Error) => void; + addedTimeMs: number; + opts: VerifySignatureOpts; + sets: {publicKey: PublicKey; signature: Uint8Array}[]; + message: Uint8Array; +}; + +export enum JobQueueItemType { + default = "default", + sameMessage = "same_message", +} + +/** + * Return count of signature sets from a JobQueueItem + */ +export function jobItemSigSets(job: JobQueueItem): number { + switch (job.type) { + case JobQueueItemType.default: + return job.sets.length; + case JobQueueItemType.sameMessage: + return 1; + } +} + +/** + * Prepare BlsWorkReq from JobQueueItem + * WARNING: May throw with untrusted user input + */ +export function jobItemWorkReq(job: JobQueueItem, format: PointFormat): BlsWorkReq { + switch (job.type) { + case JobQueueItemType.default: + return { + opts: job.opts, + sets: job.sets.map((set) => ({ + // this can throw, handled in the consumer code + publicKey: getAggregatedPubkey(set).toBytes(format), + signature: set.signature, + message: set.signingRoot, + })), + }; + case JobQueueItemType.sameMessage: + return { + opts: job.opts, + sets: [ + { + publicKey: bls.PublicKey.aggregate(job.sets.map((set) => set.publicKey)).toBytes(format), + signature: bls.Signature.aggregate( + // validate signature = true + job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true)) + ).toBytes(format), + message: job.message, + }, + ], + }; + } +} + +/** + * Convert a JobQueueItemSameMessage into multiple JobQueueItemDefault linked to the original promise + */ +export function jobItemSameMessageToMultiSet(job: JobQueueItemSameMessage): LinkedList { + // Retry each individually + // Create new jobs for each pubkey set, and Promise.all all the results + const promises: Promise[] = []; + const jobs = new LinkedList(); + + for (const set of job.sets) { + promises.push( + new Promise((resolve, reject) => { + jobs.push({ + type: JobQueueItemType.default, + resolve, + reject, + addedTimeMs: job.addedTimeMs, + opts: {batchable: false, priority: job.opts.priority}, + sets: [ + { + type: SignatureSetType.single, + pubkey: set.publicKey, + signature: set.signature, + signingRoot: job.message, + }, + ], + }); + }) + ); + } + + // Connect jobs to main job + Promise.all(promises).then(job.resolve, job.reject); + + return jobs; +} diff --git a/packages/beacon-node/src/chain/bls/multithread/types.ts b/packages/beacon-node/src/chain/bls/multithread/types.ts index d11b053ed178..cefdc799ee16 100644 --- a/packages/beacon-node/src/chain/bls/multithread/types.ts +++ b/packages/beacon-node/src/chain/bls/multithread/types.ts @@ -21,7 +21,8 @@ export enum WorkResultCode { error = "error", } -export type WorkResult = {code: WorkResultCode.success; result: R} | {code: WorkResultCode.error; error: Error}; +export type WorkResultError = {code: WorkResultCode.error; error: Error}; +export type WorkResult = {code: WorkResultCode.success; result: R} | WorkResultError; export type BlsWorkResult = { /** Ascending integer identifying the worker for metrics */ diff --git a/packages/beacon-node/src/chain/bls/singleThread.ts b/packages/beacon-node/src/chain/bls/singleThread.ts index 78f3f4bf5200..49eae40e3b8b 100644 --- a/packages/beacon-node/src/chain/bls/singleThread.ts +++ b/packages/beacon-node/src/chain/bls/singleThread.ts @@ -1,3 +1,6 @@ +import {PublicKey, Signature} from "@chainsafe/bls/types"; +import bls from "@chainsafe/bls"; +import {CoordType} from "@chainsafe/blst"; import {ISignatureSet} from "@lodestar/state-transition"; import {Metrics} from "../../metrics/index.js"; import {IBlsVerifier} from "./interface.js"; @@ -29,11 +32,53 @@ export class BlsSingleThreadVerifier implements IBlsVerifier { const endNs = process.hrtime.bigint(); const totalSec = Number(startNs - endNs) / 1e9; this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.observe(totalSec); - this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.observe(totalSec / sets.length); return isValid; } + async verifySignatureSetsSameMessage( + sets: {publicKey: PublicKey; signature: Uint8Array}[], + message: Uint8Array + ): Promise { + const startNs = process.hrtime.bigint(); + const pubkey = bls.PublicKey.aggregate(sets.map((set) => set.publicKey)); + let isAllValid = true; + // validate signature = true + const signatures = sets.map((set) => { + try { + return bls.Signature.fromBytes(set.signature, CoordType.affine, true); + } catch (_) { + // at least one set has malformed signature + isAllValid = false; + return null; + } + }); + + if (isAllValid) { + const signature = bls.Signature.aggregate(signatures as Signature[]); + isAllValid = signature.verify(pubkey, message); + } + + let result: boolean[]; + if (isAllValid) { + result = sets.map(() => true); + } else { + result = sets.map((set, i) => { + const sig = signatures[i]; + if (sig === null) { + return false; + } + return sig.verify(set.publicKey, message); + }); + } + + const endNs = process.hrtime.bigint(); + const totalSec = Number(startNs - endNs) / 1e9; + this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.observe(totalSec); + + return result; + } + async close(): Promise { // nothing to do } diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index f3a67700c53e..04915a064899 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -336,6 +336,11 @@ export function createLodestarMetrics( name: "lodestar_bls_thread_pool_success_jobs_signature_sets_count", help: "Count of total verified signature sets", }), + errorAggregateSignatureSetsCount: register.gauge<"type">({ + name: "lodestar_bls_thread_pool_error_aggregate_signature_sets_count", + help: "Count of error when aggregating pubkeys or signatures", + labelNames: ["type"], + }), errorJobsSignatureSetsCount: register.gauge({ name: "lodestar_bls_thread_pool_error_jobs_signature_sets_count", help: "Count of total error-ed signature sets", @@ -357,13 +362,15 @@ export function createLodestarMetrics( name: "lodestar_bls_thread_pool_job_groups_started_total", help: "Count of total jobs groups started in bls thread pool, job groups include +1 jobs", }), - totalJobsStarted: register.gauge({ + totalJobsStarted: register.gauge<"type">({ name: "lodestar_bls_thread_pool_jobs_started_total", help: "Count of total jobs started in bls thread pool, jobs include +1 signature sets", + labelNames: ["type"], }), - totalSigSetsStarted: register.gauge({ + totalSigSetsStarted: register.gauge<"type">({ name: "lodestar_bls_thread_pool_sig_sets_started_total", help: "Count of total signature sets started in bls thread pool, sig sets include 1 pk, msg, sig", + labelNames: ["type"], }), // Re-verifying a batch means doing double work. This number must be very low or it can be a waste of CPU resources batchRetries: register.gauge({ @@ -375,6 +382,14 @@ export function createLodestarMetrics( name: "lodestar_bls_thread_pool_batch_sigs_success_total", help: "Count of total batches that failed and had to be verified again.", }), + sameMessageRetryJobs: register.gauge({ + name: "lodestar_bls_thread_pool_same_message_jobs_retries_total", + help: "Count of total same message jobs that failed and had to be verified again.", + }), + sameMessageRetrySets: register.gauge({ + name: "lodestar_bls_thread_pool_same_message_sets_retries_total", + help: "Count of total same message sets that failed and had to be verified again.", + }), // To measure the time cost of main thread <-> worker message passing latencyToWorker: register.histogram({ name: "lodestar_bls_thread_pool_latency_to_worker", diff --git a/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts b/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts index 022d2aaacbc6..27ea9e094382 100644 --- a/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts +++ b/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts @@ -1,12 +1,13 @@ import {expect} from "chai"; import bls from "@chainsafe/bls"; +import {PublicKey} from "@chainsafe/bls/types"; import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; import {BlsMultiThreadWorkerPool} from "../../../../src/chain/bls/multithread/index.js"; import {testLogger} from "../../../utils/logger.js"; import {VerifySignatureOpts} from "../../../../src/chain/bls/interface.js"; describe("chain / bls / multithread queue", function () { - this.timeout(30 * 1000); + this.timeout(60 * 1000); const logger = testLogger(); let controller: AbortController; @@ -22,6 +23,9 @@ describe("chain / bls / multithread queue", function () { }); const sets: ISignatureSet[] = []; + const sameMessageSets: {publicKey: PublicKey; signature: Uint8Array}[] = []; + const sameMessage = Buffer.alloc(32, 100); + before("generate test data", () => { for (let i = 0; i < 3; i++) { const sk = bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1)); @@ -34,6 +38,10 @@ describe("chain / bls / multithread queue", function () { signingRoot: msg, signature: sig.toBytes(), }); + sameMessageSets.push({ + publicKey: pk, + signature: sk.sign(sameMessage).toBytes(), + }); } }); @@ -52,9 +60,10 @@ describe("chain / bls / multithread queue", function () { ): Promise { const pool = await initializePool(); - const isValidPromiseArr: Promise[] = []; + const isValidPromiseArr: Promise[] = []; for (let i = 0; i < 8; i++) { isValidPromiseArr.push(pool.verifySignatureSets(sets, verifySignatureOpts)); + isValidPromiseArr.push(pool.verifySignatureSetsSameMessage(sameMessageSets, sameMessage, verifySignatureOpts)); if (testOpts.sleep) { // Tick forward so the pool sends a job out await new Promise((r) => setTimeout(r, 5)); @@ -63,42 +72,56 @@ describe("chain / bls / multithread queue", function () { const isValidArr = await Promise.all(isValidPromiseArr); for (const [i, isValid] of isValidArr.entries()) { - expect(isValid).to.equal(true, `sig set ${i} returned invalid`); + if (i % 2 === 0) { + expect(isValid).to.equal(true, `sig set ${i} returned invalid`); + } else { + expect(isValid).to.deep.equal([true, true, true], `sig set ${i} returned invalid`); + } } } - it("Should verify multiple signatures submitted synchronously", async () => { - // Given the `setTimeout(this.runJob, 0);` all sets should be verified in a single job an worker - await testManyValidSignatures({sleep: false}); - }); + for (const priority of [true, false]) { + it(`Should verify multiple signatures submitted synchronously priority=${priority}`, async () => { + // Given the `setTimeout(this.runJob, 0);` all sets should be verified in a single job an worker + // when priority = true, jobs are executed in the reverse order + await testManyValidSignatures({sleep: false}, {priority}); + }); + } - it("Should verify multiple signatures submitted asynchronously", async () => { - // Because of the sleep, each sets submitted should be verified in a different job and worker - await testManyValidSignatures({sleep: true}); - }); + for (const priority of [true, false]) { + it(`Should verify multiple signatures submitted asynchronously priority=${priority}`, async () => { + // Because of the sleep, each sets submitted should be verified in a different job and worker + // when priority = true, jobs are executed in the reverse order + await testManyValidSignatures({sleep: true}, {priority}); + }); + } - it("Should verify multiple signatures batched", async () => { - // By setting batchable: true, 5*8 = 40 sig sets should be verified in one job, while 3*8=24 should - // be verified in another job - await testManyValidSignatures({sleep: true}, {batchable: true}); - }); + for (const priority of [true, false]) { + it(`Should verify multiple signatures batched pririty=${priority}`, async () => { + // By setting batchable: true, 5*8 = 40 sig sets should be verified in one job, while 3*8=24 should + // be verified in another job + await testManyValidSignatures({sleep: true}, {batchable: true, priority}); + }); + } - it("Should verify multiple signatures batched, first is invalid", async () => { - // If the first signature is invalid it should not make the rest throw - const pool = await initializePool(); + for (const priority of [true, false]) { + it(`Should verify multiple signatures batched, first is invalid priority=${priority}`, async () => { + // If the first signature is invalid it should not make the rest throw + const pool = await initializePool(); - const invalidSet: ISignatureSet = {...sets[0], signature: Buffer.alloc(32, 0)}; - const isInvalidPromise = pool.verifySignatureSets([invalidSet], {batchable: true}); - const isValidPromiseArr: Promise[] = []; - for (let i = 0; i < 8; i++) { - isValidPromiseArr.push(pool.verifySignatureSets(sets, {batchable: true})); - } + const invalidSet: ISignatureSet = {...sets[0], signature: Buffer.alloc(32, 0)}; + const isInvalidPromise = pool.verifySignatureSets([invalidSet], {batchable: true, priority}); + const isValidPromiseArr: Promise[] = []; + for (let i = 0; i < 8; i++) { + isValidPromiseArr.push(pool.verifySignatureSets(sets, {batchable: true})); + } - await expect(isInvalidPromise).to.rejectedWith("BLST_INVALID_SIZE"); + expect(await isInvalidPromise).to.be.false; - const isValidArr = await Promise.all(isValidPromiseArr); - for (const [i, isValid] of isValidArr.entries()) { - expect(isValid).to.equal(true, `sig set ${i} returned invalid`); - } - }); + const isValidArr = await Promise.all(isValidPromiseArr); + for (const [i, isValid] of isValidArr.entries()) { + expect(isValid).to.equal(true, `sig set ${i} returned invalid`); + } + }); + } }); diff --git a/packages/beacon-node/test/unit/chain/bls/bls.test.ts b/packages/beacon-node/test/unit/chain/bls/bls.test.ts new file mode 100644 index 000000000000..89a5e48a9db9 --- /dev/null +++ b/packages/beacon-node/test/unit/chain/bls/bls.test.ts @@ -0,0 +1,89 @@ +import bls from "@chainsafe/bls"; +import {expect} from "chai"; +import {CoordType} from "@chainsafe/blst"; +import {PublicKey} from "@chainsafe/bls/types"; +import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; +import {BlsSingleThreadVerifier} from "../../../../src/chain/bls/singleThread.js"; +import {BlsMultiThreadWorkerPool} from "../../../../lib/chain/bls/multithread/index.js"; +import {testLogger} from "../../../utils/logger.js"; + +describe("BlsVerifier ", function () { + // take time for creating thread pool + this.timeout(60 * 1000); + const numKeys = 3; + const secretKeys = Array.from({length: numKeys}, (_, i) => bls.SecretKey.fromKeygen(Buffer.alloc(32, i))); + const verifiers = [ + new BlsSingleThreadVerifier({metrics: null}), + new BlsMultiThreadWorkerPool({}, {metrics: null, logger: testLogger()}), + ]; + + for (const verifier of verifiers) { + describe(`${verifier.constructor.name} - verifySignatureSets`, () => { + let sets: ISignatureSet[]; + + beforeEach(() => { + sets = secretKeys.map((secretKey, i) => { + // different signing roots + const signingRoot = Buffer.alloc(32, i); + return { + type: SignatureSetType.single, + pubkey: secretKey.toPublicKey(), + signingRoot, + signature: secretKey.sign(signingRoot).toBytes(), + }; + }); + }); + + it("should verify all signatures", async () => { + expect(await verifier.verifySignatureSets(sets)).to.be.true; + }); + + it("should return false if at least one signature is invalid", async () => { + // signature is valid but not respective to the signing root + sets[1].signingRoot = Buffer.alloc(32, 10); + expect(await verifier.verifySignatureSets(sets)).to.be.false; + }); + + it("should return false if at least one signature is malformed", async () => { + // signature is malformed + const malformedSignature = Buffer.alloc(96, 10); + expect(() => bls.Signature.fromBytes(malformedSignature, CoordType.affine, true)).to.throws(); + sets[1].signature = malformedSignature; + expect(await verifier.verifySignatureSets(sets)).to.be.false; + }); + }); + + describe(`${verifier.constructor.name} - verifySignatureSetsSameMessage`, () => { + let sets: {publicKey: PublicKey; signature: Uint8Array}[] = []; + // same signing root for all sets + const signingRoot = Buffer.alloc(32, 100); + + beforeEach(() => { + sets = secretKeys.map((secretKey) => { + return { + publicKey: secretKey.toPublicKey(), + signature: secretKey.sign(signingRoot).toBytes(), + }; + }); + }); + + it("should verify all signatures", async () => { + expect(await verifier.verifySignatureSetsSameMessage(sets, signingRoot)).to.deep.equal([true, true, true]); + }); + + it("should return false for invalid signature", async () => { + // signature is valid but not respective to the signing root + sets[1].signature = secretKeys[1].sign(Buffer.alloc(32)).toBytes(); + expect(await verifier.verifySignatureSetsSameMessage(sets, signingRoot)).to.be.deep.equal([true, false, true]); + }); + + it("should return false for malformed signature", async () => { + // signature is malformed + const malformedSignature = Buffer.alloc(96, 10); + expect(() => bls.Signature.fromBytes(malformedSignature, CoordType.affine, true)).to.throws(); + sets[1].signature = malformedSignature; + expect(await verifier.verifySignatureSetsSameMessage(sets, signingRoot)).to.be.deep.equal([true, false, true]); + }); + }); + } +}); diff --git a/packages/beacon-node/test/unit/chain/validation/block.test.ts b/packages/beacon-node/test/unit/chain/validation/block.test.ts index 297c9c90d6cb..6ddb27fceca8 100644 --- a/packages/beacon-node/test/unit/chain/validation/block.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/block.test.ts @@ -44,7 +44,12 @@ describe("gossip block validation", function () { verifySignature = sinon.stub(); verifySignature.resolves(true); - chain.bls = {verifySignatureSets: verifySignature, close: () => Promise.resolve(), canAcceptWork: () => true}; + chain.bls = { + verifySignatureSets: verifySignature, + verifySignatureSetsSameMessage: () => Promise.resolve([true]), + close: () => Promise.resolve(), + canAcceptWork: () => true, + }; forkChoice.getFinalizedCheckpoint.returns({epoch: 0, root: ZERO_HASH, rootHex: ""}); diff --git a/packages/beacon-node/test/utils/mocks/bls.ts b/packages/beacon-node/test/utils/mocks/bls.ts index 57e84d509fc7..e90287dad524 100644 --- a/packages/beacon-node/test/utils/mocks/bls.ts +++ b/packages/beacon-node/test/utils/mocks/bls.ts @@ -1,3 +1,4 @@ +import {PublicKey} from "@chainsafe/bls/types"; import {IBlsVerifier} from "../../../src/chain/bls/index.js"; export class BlsVerifierMock implements IBlsVerifier { @@ -7,6 +8,10 @@ export class BlsVerifierMock implements IBlsVerifier { return this.isValidResult; } + async verifySignatureSetsSameMessage(sets: {publicKey: PublicKey; signature: Uint8Array}[]): Promise { + return sets.map(() => this.isValidResult); + } + async close(): Promise { // } From 03c36e28f7352510d29f6b837c756933ddaee4b7 Mon Sep 17 00:00:00 2001 From: Cayman Date: Fri, 21 Jul 2023 10:22:39 -0400 Subject: [PATCH 78/96] refactor(beacon-node): simplify libp2p init (#5774) * refactor(beacon-node): simplify libp2p init * fix: update discv5 bootnode parsing * feat: throw on invalid bootnodes * chore: use valid enrs in tests * fix: validate enrs coming from bootnodesFile * fix: beacon args handler test --- .../src/network/core/networkCore.ts | 2 +- packages/beacon-node/src/network/index.ts | 2 +- .../{nodejs/bundle.ts => libp2p/index.ts} | 103 ++++++++++-------- .../src/network/{nodejs => libp2p}/noise.ts | 0 .../beacon-node/src/network/nodejs/index.ts | 2 - .../beacon-node/src/network/nodejs/util.ts | 68 ------------ .../test/unit/network/util.test.ts | 51 +-------- packages/beacon-node/test/utils/network.ts | 10 +- packages/cli/src/networks/index.ts | 8 ++ .../src/options/beaconNodeOptions/network.ts | 16 ++- packages/cli/test/unit/cmds/beacon.test.ts | 10 +- .../unit/options/beaconNodeOptions.test.ts | 8 +- 12 files changed, 104 insertions(+), 176 deletions(-) rename packages/beacon-node/src/network/{nodejs/bundle.ts => libp2p/index.ts} (54%) rename packages/beacon-node/src/network/{nodejs => libp2p}/noise.ts (100%) delete mode 100644 packages/beacon-node/src/network/nodejs/index.ts delete mode 100644 packages/beacon-node/src/network/nodejs/util.ts diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts index c9c62a45304c..d2545a4e758c 100644 --- a/packages/beacon-node/src/network/core/networkCore.ts +++ b/packages/beacon-node/src/network/core/networkCore.ts @@ -22,7 +22,7 @@ import {FORK_EPOCH_LOOKAHEAD, getActiveForks} from "../forks.js"; import {NetworkOptions} from "../options.js"; import {CommitteeSubscription, IAttnetsService} from "../subnets/interface.js"; import {MetadataController} from "../metadata.js"; -import {createNodeJsLibp2p} from "../nodejs/util.js"; +import {createNodeJsLibp2p} from "../libp2p/index.js"; import {PeersData} from "../peers/peersData.js"; import {PeerAction, PeerRpcScoreStore, PeerScoreStats} from "../peers/index.js"; import {getConnectionsMap} from "../util.js"; diff --git a/packages/beacon-node/src/network/index.ts b/packages/beacon-node/src/network/index.ts index 491deb57fb5e..073d1ddbfbc1 100644 --- a/packages/beacon-node/src/network/index.ts +++ b/packages/beacon-node/src/network/index.ts @@ -1,7 +1,7 @@ export * from "./events.js"; export * from "./interface.js"; export * from "./network.js"; -export * from "./nodejs/index.js"; +export * from "./libp2p/index.js"; export * from "./gossip/index.js"; export * from "./reqresp/ReqRespBeaconNode.js"; export * from "./util.js"; diff --git a/packages/beacon-node/src/network/nodejs/bundle.ts b/packages/beacon-node/src/network/libp2p/index.ts similarity index 54% rename from packages/beacon-node/src/network/nodejs/bundle.ts rename to packages/beacon-node/src/network/libp2p/index.ts index 6243ffd0364b..4540cae67974 100644 --- a/packages/beacon-node/src/network/nodejs/bundle.ts +++ b/packages/beacon-node/src/network/libp2p/index.ts @@ -1,72 +1,93 @@ -import {createLibp2p} from "libp2p"; +import {PeerId} from "@libp2p/interface-peer-id"; +import {Registry} from "prom-client"; +import {ENR} from "@chainsafe/discv5"; +import type {Components} from "libp2p/components"; import {identifyService} from "libp2p/identify"; -import {tcp} from "@libp2p/tcp"; -import {mplex} from "@libp2p/mplex"; import {bootstrap} from "@libp2p/bootstrap"; import {mdns} from "@libp2p/mdns"; -import {PeerId} from "@libp2p/interface-peer-id"; -import {Datastore} from "interface-datastore"; -import type {PeerDiscovery} from "@libp2p/interface-peer-discovery"; -import type {Components} from "libp2p/components"; +import {createLibp2p} from "libp2p"; +import {mplex} from "@libp2p/mplex"; import {prometheusMetrics} from "@libp2p/prometheus-metrics"; -import {Registry} from "prom-client"; +import {tcp} from "@libp2p/tcp"; +import {defaultNetworkOptions, NetworkOptions} from "../options.js"; +import {Eth2PeerDataStore} from "../peers/datastore.js"; import {Libp2p} from "../interface.js"; import {createNoise} from "./noise.js"; -export type Libp2pOptions = { - peerId: PeerId; - addresses: { - listen: string[]; - announce?: string[]; - }; - datastore?: Datastore; - peerDiscovery?: ((components: Components) => PeerDiscovery)[]; - bootMultiaddrs?: string[]; - maxConnections?: number; - minConnections?: number; +export type NodeJsLibp2pOpts = { + peerStoreDir?: string; + disablePeerDiscovery?: boolean; metrics?: boolean; metricsRegistry?: Registry; - lodestarVersion?: string; - hideAgentVersion?: boolean; - mdns?: boolean; }; -export async function createNodejsLibp2p(options: Libp2pOptions): Promise { +export async function getDiscv5Multiaddrs(bootEnrs: string[]): Promise { + const bootMultiaddrs = []; + for (const enrStr of bootEnrs) { + const enr = ENR.decodeTxt(enrStr); + const multiaddrWithPeerId = (await enr.getFullMultiaddr("tcp"))?.toString(); + if (multiaddrWithPeerId) { + bootMultiaddrs.push(multiaddrWithPeerId); + } + } + return bootMultiaddrs; +} + +export async function createNodeJsLibp2p( + peerId: PeerId, + networkOpts: Partial = {}, + nodeJsLibp2pOpts: NodeJsLibp2pOpts = {} +): Promise { + const localMultiaddrs = networkOpts.localMultiaddrs || defaultNetworkOptions.localMultiaddrs; + const {peerStoreDir, disablePeerDiscovery} = nodeJsLibp2pOpts; + + let datastore: undefined | Eth2PeerDataStore = undefined; + if (peerStoreDir) { + datastore = new Eth2PeerDataStore(peerStoreDir); + await datastore.open(); + } + const peerDiscovery = []; - if (options.peerDiscovery) { - peerDiscovery.push(...options.peerDiscovery); - } else { - if ((options.bootMultiaddrs?.length ?? 0) > 0) { - peerDiscovery.push(bootstrap({list: options.bootMultiaddrs ?? []})); + if (!disablePeerDiscovery) { + const bootMultiaddrs = [ + ...(networkOpts.bootMultiaddrs ?? defaultNetworkOptions.bootMultiaddrs ?? []), + // Append discv5.bootEnrs to bootMultiaddrs if requested + ...(networkOpts.connectToDiscv5Bootnodes ? await getDiscv5Multiaddrs(networkOpts.discv5?.bootEnrs ?? []) : []), + ]; + + if ((bootMultiaddrs.length ?? 0) > 0) { + peerDiscovery.push(bootstrap({list: bootMultiaddrs})); } - if (options.mdns) { + + if (networkOpts.mdns) { peerDiscovery.push(mdns()); } } + return createLibp2p({ - peerId: options.peerId, + peerId, addresses: { - listen: options.addresses.listen, - announce: options.addresses.announce || [], + listen: localMultiaddrs, + announce: [], }, connectionEncryption: [createNoise()], // Reject connections when the server's connection count gets high transports: [ tcp({ - maxConnections: options.maxConnections, + maxConnections: networkOpts.maxPeers, closeServerOnMaxConnections: { - closeAbove: options.maxConnections ?? Infinity, - listenBelow: options.maxConnections ?? Infinity, + closeAbove: networkOpts.maxPeers ?? Infinity, + listenBelow: networkOpts.maxPeers ?? Infinity, }, }), ], streamMuxers: [mplex({maxInboundStreams: 256})], peerDiscovery, - metrics: options.metrics + metrics: nodeJsLibp2pOpts.metrics ? prometheusMetrics({ collectDefaultMetrics: false, preserveExistingMetrics: true, - registry: options.metricsRegistry, + registry: nodeJsLibp2pOpts.metricsRegistry, }) : undefined, connectionManager: { @@ -81,14 +102,10 @@ export async function createNodejsLibp2p(options: Libp2pOptions): Promise, - networkOpts: Partial = {}, - nodeJsLibp2pOpts: NodeJsLibp2pOpts = {} -): Promise { - const peerId = await Promise.resolve(peerIdOrPromise); - const localMultiaddrs = networkOpts.localMultiaddrs || defaultNetworkOptions.localMultiaddrs; - const bootMultiaddrs = networkOpts.bootMultiaddrs || defaultNetworkOptions.bootMultiaddrs; - const {peerStoreDir, disablePeerDiscovery} = nodeJsLibp2pOpts; - - let datastore: undefined | Eth2PeerDataStore = undefined; - if (peerStoreDir) { - datastore = new Eth2PeerDataStore(peerStoreDir); - await datastore.open(); - } - - // Append discv5.bootEnrs to bootMultiaddrs if requested - if (networkOpts.connectToDiscv5Bootnodes) { - if (!networkOpts.bootMultiaddrs) { - networkOpts.bootMultiaddrs = []; - } - for (const enrOrStr of networkOpts.discv5?.bootEnrs ?? []) { - const enr = typeof enrOrStr === "string" ? ENR.decodeTxt(enrOrStr) : enrOrStr; - const fullMultiAddr = await enr.getFullMultiaddr("tcp"); - const multiaddrWithPeerId = fullMultiAddr?.toString(); - if (multiaddrWithPeerId) { - networkOpts.bootMultiaddrs.push(multiaddrWithPeerId); - } - } - } - - return _createNodejsLibp2p({ - peerId, - addresses: {listen: localMultiaddrs}, - datastore, - bootMultiaddrs: bootMultiaddrs, - maxConnections: networkOpts.maxPeers, - minConnections: networkOpts.targetPeers, - // If peer discovery is enabled let the default in NodejsNode - peerDiscovery: disablePeerDiscovery ? [] : undefined, - metrics: nodeJsLibp2pOpts.metrics, - metricsRegistry: nodeJsLibp2pOpts.metricsRegistry, - lodestarVersion: networkOpts.version, - hideAgentVersion: networkOpts.private, - mdns: networkOpts.mdns, - }); -} diff --git a/packages/beacon-node/test/unit/network/util.test.ts b/packages/beacon-node/test/unit/network/util.test.ts index 1ee056ec0786..b01c4100e974 100644 --- a/packages/beacon-node/test/unit/network/util.test.ts +++ b/packages/beacon-node/test/unit/network/util.test.ts @@ -1,10 +1,7 @@ import {expect} from "chai"; -import {createSecp256k1PeerId} from "@libp2p/peer-id-factory"; -import {generateKeypair, KeypairType, SignableENR} from "@chainsafe/discv5"; import {config} from "@lodestar/config/default"; import {ForkName} from "@lodestar/params"; -import {defaultNetworkOptions} from "../../../src/network/options.js"; -import {createNodeJsLibp2p} from "../../../src/network/index.js"; +import {getDiscv5Multiaddrs} from "../../../src/network/libp2p/index.js"; import {getCurrentAndNextFork} from "../../../src/network/forks.js"; describe("getCurrentAndNextFork", function () { @@ -36,33 +33,13 @@ describe("getCurrentAndNextFork", function () { }); }); -describe("createNodeJsLibp2p", () => { +describe("getDiscv5Multiaddrs", () => { it("should extract bootMultiaddrs from enr with tcp", async function () { this.timeout(0); - const peerId = await createSecp256k1PeerId(); const enrWithTcp = [ "enr:-LK4QDiPGwNomqUqNDaM3iHYvtdX7M5qngson6Qb2xGIg1LwC8-Nic0aQwO0rVbJt5xp32sRE3S1YqvVrWO7OgVNv0kBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpA7CIeVAAAgCf__________gmlkgnY0gmlwhBKNA4qJc2VjcDI1NmsxoQKbBS4ROQ_sldJm5tMgi36qm5I5exKJFb4C8dDVS_otAoN0Y3CCIyiDdWRwgiMo", ]; - const bootMultiaddrs: string[] = []; - const keypair = generateKeypair(KeypairType.Secp256k1); - await createNodeJsLibp2p( - peerId, - { - connectToDiscv5Bootnodes: true, - discv5: { - enr: SignableENR.createV4(keypair).encodeTxt(), - bindAddrs: { - ip4: "/ip4/127.0.0.1/udp/0", - }, - bootEnrs: enrWithTcp, - }, - bootMultiaddrs, - localMultiaddrs: ["/ip4/127.0.0.1/tcp/0"], - targetPeers: defaultNetworkOptions.targetPeers, - maxPeers: defaultNetworkOptions.maxPeers, - }, - {disablePeerDiscovery: true} - ); + const bootMultiaddrs = await getDiscv5Multiaddrs(enrWithTcp); expect(bootMultiaddrs.length).to.be.equal(1); expect(bootMultiaddrs[0]).to.be.equal( "/ip4/18.141.3.138/tcp/9000/p2p/16Uiu2HAm5rokhpCBU7yBJHhMKXZ1xSVWwUcPMrzGKvU5Y7iBkmuK" @@ -71,30 +48,10 @@ describe("createNodeJsLibp2p", () => { it("should not extract bootMultiaddrs from enr without tcp", async function () { this.timeout(0); - const peerId = await createSecp256k1PeerId(); const enrWithoutTcp = [ "enr:-Ku4QCFQW96tEDYPjtaueW3WIh1CB0cJnvw_ibx5qIFZGqfLLj-QajMX6XwVs2d4offuspwgH3NkIMpWtCjCytVdlywGh2F0dG5ldHOIEAIAAgABAUyEZXRoMpCi7FS9AQAAAAAiAQAAAAAAgmlkgnY0gmlwhFA4VK6Jc2VjcDI1NmsxoQNGH1sJJS86-0x9T7qQewz9Wn9zlp6bYxqqrR38JQ49yIN1ZHCCIyg", ]; - const bootMultiaddrs: string[] = []; - const keypair = generateKeypair(KeypairType.Secp256k1); - await createNodeJsLibp2p( - peerId, - { - connectToDiscv5Bootnodes: true, - discv5: { - enr: SignableENR.createV4(keypair).encodeTxt(), - bindAddrs: { - ip4: "/ip4/127.0.0.1/udp/0", - }, - bootEnrs: enrWithoutTcp, - }, - bootMultiaddrs, - localMultiaddrs: ["/ip4/127.0.0.1/tcp/0"], - targetPeers: defaultNetworkOptions.targetPeers, - maxPeers: defaultNetworkOptions.maxPeers, - }, - {disablePeerDiscovery: true} - ); + const bootMultiaddrs = await getDiscv5Multiaddrs(enrWithoutTcp); expect(bootMultiaddrs.length).to.be.equal(0); }); }); diff --git a/packages/beacon-node/test/utils/network.ts b/packages/beacon-node/test/utils/network.ts index a514335594db..89b175c3afaa 100644 --- a/packages/beacon-node/test/utils/network.ts +++ b/packages/beacon-node/test/utils/network.ts @@ -12,7 +12,7 @@ import { NetworkInitModules, getReqRespHandlers, } from "../../src/network/index.js"; -import {createNodejsLibp2p, Libp2pOptions} from "../../src/network/nodejs/index.js"; +import {createNodeJsLibp2p} from "../../src/network/libp2p/index.js"; import {Libp2p} from "../../src/network/interface.js"; import {GetReqRespHandlerFn} from "../../src/network/reqresp/types.js"; import {Eth1ForBlockProductionDisabled} from "../../src/eth1/index.js"; @@ -26,13 +26,9 @@ import {getStubbedBeaconDb} from "./mocks/db.js"; import {ClockStatic} from "./clock.js"; import {createCachedBeaconStateTest} from "./cachedBeaconState.js"; -export async function createNode(multiaddr: string, inPeerId?: PeerId, opts?: Partial): Promise { +export async function createNode(multiaddr: string, inPeerId?: PeerId): Promise { const peerId = inPeerId || (await createSecp256k1PeerId()); - return createNodejsLibp2p({ - peerId, - addresses: {listen: [multiaddr]}, - ...opts, - }); + return createNodeJsLibp2p(peerId, {localMultiaddrs: [multiaddr]}); } export async function createNetworkModules( diff --git a/packages/cli/src/networks/index.ts b/packages/cli/src/networks/index.ts index 049cca9b002d..d9c5812bc5b4 100644 --- a/packages/cli/src/networks/index.ts +++ b/packages/cli/src/networks/index.ts @@ -1,5 +1,6 @@ import fs from "node:fs"; import got from "got"; +import {ENR} from "@chainsafe/discv5"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ApiError, getClient} from "@lodestar/api"; import {getStateTypeFromBytes} from "@lodestar/beacon-node"; @@ -118,6 +119,13 @@ export function readBootnodes(bootnodesFilePath: string): string[] { const bootnodesFile = fs.readFileSync(bootnodesFilePath, "utf8"); const bootnodes = parseBootnodesFile(bootnodesFile); + for (const enrStr of bootnodes) { + try { + ENR.decodeTxt(enrStr); + } catch (e) { + throw new Error(`Invalid ENR found in ${bootnodesFilePath}:\n ${enrStr}`); + } + } if (bootnodes.length === 0) { throw new Error(`No bootnodes found on file ${bootnodesFilePath}`); diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 6f30c3aabe76..be71619f4c8f 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -1,4 +1,5 @@ import {multiaddr} from "@multiformats/multiaddr"; +import {ENR} from "@chainsafe/discv5"; import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; import {CliCommandOptions, YargsError} from "../../util/index.js"; @@ -104,6 +105,18 @@ export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { } // Set discv5 opts to null to disable only if explicitly disabled const enableDiscv5 = args["discv5"] ?? true; + + // TODO: Okay to set to empty array? + const bootEnrs = args["bootnodes"] ?? []; + // throw if user-provided enrs are invalid + for (const enrStr of bootEnrs) { + try { + ENR.decodeTxt(enrStr); + } catch (e) { + throw new YargsError(`Provided ENR in bootnodes is invalid:\n ${enrStr}`); + } + } + return { discv5: enableDiscv5 ? { @@ -112,8 +125,7 @@ export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { ip4: bindMu as string, ip6: bindMu6, }, - // TODO: Okay to set to empty array? - bootEnrs: args["bootnodes"] ?? [], + bootEnrs, // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any enr: undefined as any, } diff --git a/packages/cli/test/unit/cmds/beacon.test.ts b/packages/cli/test/unit/cmds/beacon.test.ts index 02aa91f2d4b7..08367ad01309 100644 --- a/packages/cli/test/unit/cmds/beacon.test.ts +++ b/packages/cli/test/unit/cmds/beacon.test.ts @@ -19,8 +19,10 @@ describe("cmds / beacon / args handler", () => { process.env.SKIP_FETCH_NETWORK_BOOTNODES = "true"; it("Merge bootnodes from file and CLI arg", async () => { - const enr1 = "enr:-AAKG4QOWkRj"; - const enr2 = "enr:-BBBBBBW4gMj"; + const enr1 = + "enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA"; + const enr2 = + "enr:-KG4QDyytgmE4f7AnvW-ZaUOIi9i79qX4JwjRAiXBZCU65wOfBu-3Nb5I7b_Rmg3KCOcZM_C3y5pg7EBU5XGrcLTduQEhGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQ2_DUbiXNlY3AyNTZrMaEDKnz_-ps3UUOfHWVYaskI5kWYO_vtYMGYCQRAR3gHDouDdGNwgiMog3VkcIIjKA"; const bootnodesFile = path.join(testFilesDir, "bootnodesFile.txt"); fs.writeFileSync(bootnodesFile, enr1); @@ -30,7 +32,9 @@ describe("cmds / beacon / args handler", () => { bootnodesFile, }); - expect(options.network.discv5?.bootEnrs?.sort().slice(0, 2)).to.deep.equal([enr1, enr2]); + const bootEnrs = options.network.discv5?.bootEnrs ?? []; + expect(bootEnrs.includes(enr1)).to.be.true; + expect(bootEnrs.includes(enr2)).to.be.true; }); it("Over-write ENR fields", async () => { diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index ee694bc2f173..40f820bac57e 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -69,7 +69,9 @@ describe("options / beaconNodeOptions", () => { listenAddress: "127.0.0.1", port: 9001, discoveryPort: 9002, - bootnodes: ["enr:-somedata"], + bootnodes: [ + "enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA", + ], targetPeers: 25, deterministicLongLivedAttnets: true, subscribeAllSubnets: true, @@ -169,7 +171,9 @@ describe("options / beaconNodeOptions", () => { bindAddrs: { ip4: "/ip4/127.0.0.1/udp/9002", }, - bootEnrs: ["enr:-somedata"], + bootEnrs: [ + "enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA", + ], }, maxPeers: 30, targetPeers: 25, From 325734518cf024869fb8a7d33cb75b665438a83b Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 21 Jul 2023 23:16:34 +0200 Subject: [PATCH 79/96] fix: prevent eventstream errors on shutdown or client aborts (#5784) --- packages/api/src/beacon/server/events.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/api/src/beacon/server/events.ts b/packages/api/src/beacon/server/events.ts index 0a20e57e908a..fa8b857f8779 100644 --- a/packages/api/src/beacon/server/events.ts +++ b/packages/api/src/beacon/server/events.ts @@ -18,7 +18,13 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR const controller = new AbortController(); try { - // Add injected headers from other pluggins. This is required for fastify-cors for example + // Prevent Fastify from sending the response, this is recommended before writing to the `.raw` stream + // and avoids "Cannot set headers after they are sent to the client" errors during shutdown or client aborts. + // See https://github.com/fastify/fastify/issues/3979, https://github.com/ChainSafe/lodestar/issues/5783 + // eslint-disable-next-line @typescript-eslint/no-floating-promises + res.hijack(); + + // Add injected headers from other plugins. This is required for fastify-cors for example // From: https://github.com/NodeFactoryIo/fastify-sse-v2/blob/b1686a979fbf655fb9936c0560294a0c094734d4/src/plugin.ts Object.entries(res.getHeaders()).forEach(([key, value]) => { if (value !== undefined) res.raw.setHeader(key, value); @@ -37,9 +43,6 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR await new Promise((resolve, reject) => { void api.eventstream(req.query.topics, controller.signal, (event) => { try { - // If the request is already aborted, we don't need to send any more events. - if (req.raw.destroyed) return; - const data = eventSerdes.toJson(event); res.raw.write(serializeSSEEvent({event: event.type, data})); } catch (e) { From 919e79737199b87c48d74beba8ab6f41a4118f9e Mon Sep 17 00:00:00 2001 From: g11tech Date: Mon, 24 Jul 2023 15:26:19 +0530 Subject: [PATCH 80/96] feat: enable commented deneb spec tests (#5789) --- packages/beacon-node/test/spec/presets/index.test.ts | 5 ----- packages/params/test/e2e/ensure-config-is-synced.test.ts | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/beacon-node/test/spec/presets/index.test.ts b/packages/beacon-node/test/spec/presets/index.test.ts index 1e42ddde4725..c45f859ec1e5 100644 --- a/packages/beacon-node/test/spec/presets/index.test.ts +++ b/packages/beacon-node/test/spec/presets/index.test.ts @@ -37,11 +37,6 @@ const skipOpts: SkipOpts = { skippedPrefixes: [ "capella/light_client/single_merkle_proof/BeaconBlockBody", "deneb/light_client/single_merkle_proof/BeaconBlockBody", - // TODO: deneb - // Deneb signed voluntary exits will not be valid so skipping this - // To be cleaned up with the spec version update - "deneb/operations/voluntary_exit/", - "deneb/random/random", ], }; diff --git a/packages/params/test/e2e/ensure-config-is-synced.test.ts b/packages/params/test/e2e/ensure-config-is-synced.test.ts index 2a617c65fe16..6adea922f37a 100644 --- a/packages/params/test/e2e/ensure-config-is-synced.test.ts +++ b/packages/params/test/e2e/ensure-config-is-synced.test.ts @@ -8,7 +8,7 @@ import {loadConfigYaml} from "../yaml.js"; // Not e2e, but slow. Run with e2e tests /** https://github.com/ethereum/consensus-specs/releases */ -const specConfigCommit = "v1.4.0-alpha.3"; +const specConfigCommit = "v1.4.0-beta.0"; describe("Ensure config is synced", function () { this.timeout(60 * 1000); From 7f19831f0fc7d807003e1b6a08feb42d3234e971 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 24 Jul 2023 13:56:23 +0200 Subject: [PATCH 81/96] fix: eventstream listen on socket for close/end event (#5795) * Remove res.hijack() * Listen on socket for close/end events --- packages/api/src/beacon/server/events.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/api/src/beacon/server/events.ts b/packages/api/src/beacon/server/events.ts index fa8b857f8779..3e45b104ecb5 100644 --- a/packages/api/src/beacon/server/events.ts +++ b/packages/api/src/beacon/server/events.ts @@ -18,12 +18,6 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR const controller = new AbortController(); try { - // Prevent Fastify from sending the response, this is recommended before writing to the `.raw` stream - // and avoids "Cannot set headers after they are sent to the client" errors during shutdown or client aborts. - // See https://github.com/fastify/fastify/issues/3979, https://github.com/ChainSafe/lodestar/issues/5783 - // eslint-disable-next-line @typescript-eslint/no-floating-promises - res.hijack(); - // Add injected headers from other plugins. This is required for fastify-cors for example // From: https://github.com/NodeFactoryIo/fastify-sse-v2/blob/b1686a979fbf655fb9936c0560294a0c094734d4/src/plugin.ts Object.entries(res.getHeaders()).forEach(([key, value]) => { @@ -54,8 +48,8 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR // In that case the BeaconNode class will call server.close() and end this connection. // The client may disconnect and we need to clean the subscriptions. - req.raw.once("close", () => resolve()); - req.raw.once("end", () => resolve()); + req.socket.once("close", () => resolve()); + req.socket.once("end", () => resolve()); req.raw.once("error", (err) => { if ((err as unknown as {code: string}).code === "ECONNRESET") { return reject(new ErrorAborted()); From c0aa75280c5bf7f3fd1a8116f2bfc98b3ffc413a Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Mon, 24 Jul 2023 18:54:25 +0200 Subject: [PATCH 82/96] fix: contract eth_call bug and made some improvements (#5785) * Fix some regressions for prover * Fix package export assertion for nested conditional exports * Improve the exports for the package * Fix the http agent for https * Fix e2e tests --- packages/prover/package.json | 6 ++- packages/prover/src/browser/index.ts | 3 ++ packages/prover/src/constants.ts | 2 +- packages/prover/src/index.ts | 2 + packages/prover/src/index.web.ts | 2 - packages/prover/src/interfaces.ts | 1 + packages/prover/src/proof_provider/index.ts | 1 + packages/prover/src/utils/evm.ts | 15 +++++- packages/prover/src/utils/json_rpc.ts | 16 +++++-- packages/prover/src/utils/rpc.ts | 2 +- .../prover/src/verified_requests/eth_call.ts | 8 +++- .../src/verified_requests/eth_estimateGas.ts | 8 +++- .../src/verified_requests/eth_getBalance.ts | 8 +++- .../verified_requests/eth_getBlockByHash.ts | 8 +++- .../verified_requests/eth_getBlockByNumber.ts | 11 ++++- .../src/verified_requests/eth_getCode.ts | 13 +++-- .../eth_getTransactionCount.ts | 11 ++++- packages/prover/src/web3_proxy.ts | 3 +- .../test/e2e/web3_batch_request.test.ts | 3 +- packages/prover/test/mocks/request_handler.ts | 1 + .../unit/verified_requests/eth_call.test.ts | 5 +- .../verified_requests/eth_estimateGas.test.ts | 5 +- .../verified_requests/eth_getBalance.test.ts | 5 +- .../eth_getBlockByHash.test.ts | 9 ++-- .../eth_getBlockByNumber.test.ts | 18 +++++-- .../verified_requests/eth_getCode.test.ts | 5 +- .../eth_getTransactionCount.test.ts | 8 +++- .../prover/test/unit/web3_provider.test.ts | 4 +- scripts/assert_exports.mjs | 47 ++++++++++--------- 29 files changed, 158 insertions(+), 72 deletions(-) create mode 100644 packages/prover/src/browser/index.ts delete mode 100644 packages/prover/src/index.web.ts create mode 100644 packages/prover/src/proof_provider/index.ts diff --git a/packages/prover/package.json b/packages/prover/package.json index 171d0853da66..88d0d7d8a317 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -15,8 +15,10 @@ "type": "module", "exports": { ".": { - "import": "./lib/index.js", - "browser": "./lib/index.web.js" + "import": "./lib/index.js" + }, + "./browser": { + "import": "./lib/browser/index.js" } }, "bin": { diff --git a/packages/prover/src/browser/index.ts b/packages/prover/src/browser/index.ts new file mode 100644 index 000000000000..6bce2e428088 --- /dev/null +++ b/packages/prover/src/browser/index.ts @@ -0,0 +1,3 @@ +export * from "../interfaces.js"; +export * from "../proof_provider/index.js"; +export {createVerifiedExecutionProvider} from "../web3_provider.js"; diff --git a/packages/prover/src/constants.ts b/packages/prover/src/constants.ts index d9e4b5dbfa43..5a9eefd0f3ca 100644 --- a/packages/prover/src/constants.ts +++ b/packages/prover/src/constants.ts @@ -1,6 +1,6 @@ // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#configuration export const MAX_REQUEST_LIGHT_CLIENT_UPDATES = 128; export const MAX_PAYLOAD_HISTORY = 32; -export const UNVERIFIED_RESPONSE_CODE = -33091; +export const VERIFICATION_FAILED_RESPONSE_CODE = -33091; export const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; export const DEFAULT_PROXY_REQUEST_TIMEOUT = 3000; diff --git a/packages/prover/src/index.ts b/packages/prover/src/index.ts index 59390455dc2e..daf91784cecc 100644 --- a/packages/prover/src/index.ts +++ b/packages/prover/src/index.ts @@ -1,3 +1,5 @@ export * from "./interfaces.js"; +export * from "./proof_provider/index.js"; export {createVerifiedExecutionProvider} from "./web3_provider.js"; export {createVerifiedExecutionProxy} from "./web3_proxy.js"; +export {isVerificationFailedError} from "./utils/json_rpc.js"; diff --git a/packages/prover/src/index.web.ts b/packages/prover/src/index.web.ts deleted file mode 100644 index bdde6e5542d3..000000000000 --- a/packages/prover/src/index.web.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./interfaces.js"; -export {createVerifiedExecutionProvider} from "./web3_provider.js"; diff --git a/packages/prover/src/interfaces.ts b/packages/prover/src/interfaces.ts index 6d00a7c47e6d..30558f149596 100644 --- a/packages/prover/src/interfaces.ts +++ b/packages/prover/src/interfaces.ts @@ -5,6 +5,7 @@ import {ProofProvider} from "./proof_provider/proof_provider.js"; import {JsonRpcRequest, JsonRpcRequestOrBatch, JsonRpcResponse, JsonRpcResponseOrBatch} from "./types.js"; import {ELRpc} from "./utils/rpc.js"; +export {NetworkName} from "@lodestar/config/networks"; export enum LCTransport { Rest = "Rest", P2P = "P2P", diff --git a/packages/prover/src/proof_provider/index.ts b/packages/prover/src/proof_provider/index.ts new file mode 100644 index 000000000000..d64938f13e6e --- /dev/null +++ b/packages/prover/src/proof_provider/index.ts @@ -0,0 +1 @@ +export * from "./proof_provider.js"; diff --git a/packages/prover/src/utils/evm.ts b/packages/prover/src/utils/evm.ts index 7d602a4520bf..eb0296752ffe 100644 --- a/packages/prover/src/utils/evm.ts +++ b/packages/prover/src/utils/evm.ts @@ -84,8 +84,19 @@ export async function getVMWithState({ const batchRequests = []; for (const [address, storageKeys] of Object.entries(storageKeysMap)) { - batchRequests.push({jsonrpc: "2.0", method: "eth_getProof", params: [address, storageKeys, blockHashHex]}); - batchRequests.push({jsonrpc: "2.0", method: "eth_getCode", params: [address, blockHashHex]}); + batchRequests.push({ + jsonrpc: "2.0", + id: rpc.getRequestId(), + method: "eth_getProof", + params: [address, storageKeys, blockHashHex], + }); + + batchRequests.push({ + jsonrpc: "2.0", + id: rpc.getRequestId(), + method: "eth_getCode", + params: [address, blockHashHex], + }); } // If all responses are valid then we will have even number of responses diff --git a/packages/prover/src/utils/json_rpc.ts b/packages/prover/src/utils/json_rpc.ts index cde220b597d3..74727c198122 100644 --- a/packages/prover/src/utils/json_rpc.ts +++ b/packages/prover/src/utils/json_rpc.ts @@ -1,5 +1,5 @@ import {Logger} from "@lodestar/logger"; -import {UNVERIFIED_RESPONSE_CODE} from "../constants.js"; +import {VERIFICATION_FAILED_RESPONSE_CODE} from "../constants.js"; import { JsonRpcErrorPayload, JsonRpcNotificationPayload, @@ -44,18 +44,26 @@ export function getResponseForRequest( throw new Error("Either result or error must be defined."); } -export function getErrorResponseForUnverifiedRequest( +export function getVerificationFailedMessage(method: string): string { + return `verification for '${method}' request failed.`; +} + +export function isVerificationFailedError

(payload: JsonRpcResponseWithErrorPayload

): boolean { + return !isValidResponsePayload(payload) && payload.error.code === VERIFICATION_FAILED_RESPONSE_CODE; +} + +export function getErrorResponseForRequestWithFailedVerification( payload: JsonRpcRequest

, message: string, data?: D ): JsonRpcResponseWithErrorPayload { return isNullish(data) ? (getResponseForRequest(payload, undefined, { - code: UNVERIFIED_RESPONSE_CODE, + code: VERIFICATION_FAILED_RESPONSE_CODE, message, }) as JsonRpcResponseWithErrorPayload) : (getResponseForRequest(payload, undefined, { - code: UNVERIFIED_RESPONSE_CODE, + code: VERIFICATION_FAILED_RESPONSE_CODE, message, data, }) as JsonRpcResponseWithErrorPayload); diff --git a/packages/prover/src/utils/rpc.ts b/packages/prover/src/utils/rpc.ts index 5fb8078f228c..5feee3332c4a 100644 --- a/packages/prover/src/utils/rpc.ts +++ b/packages/prover/src/utils/rpc.ts @@ -98,7 +98,7 @@ export class ELRpc { } } - private getRequestId(): string { + getRequestId(): string { // TODO: Find better way to generate random id return (Math.random() * 10000).toFixed(0); } diff --git a/packages/prover/src/verified_requests/eth_call.ts b/packages/prover/src/verified_requests/eth_call.ts index 4b0ea115245c..b28bd222c568 100644 --- a/packages/prover/src/verified_requests/eth_call.ts +++ b/packages/prover/src/verified_requests/eth_call.ts @@ -2,7 +2,11 @@ import {ELVerifiedRequestHandler} from "../interfaces.js"; import {ELApiParams, ELApiReturn} from "../types.js"; import {bufferToHex} from "../utils/conversion.js"; import {createVM, executeVMCall, getVMWithState} from "../utils/evm.js"; -import {getResponseForRequest, getErrorResponseForUnverifiedRequest} from "../utils/json_rpc.js"; +import { + getResponseForRequest, + getErrorResponseForRequestWithFailedVerification, + getVerificationFailedMessage, +} from "../utils/json_rpc.js"; // eslint-disable-next-line @typescript-eslint/naming-convention export const eth_call: ELVerifiedRequestHandler = async ({ @@ -42,6 +46,6 @@ export const eth_call: ELVerifiedRequestHandler = async ({ @@ -19,5 +23,5 @@ export const eth_getBalance: ELVerifiedRequestHandler<[address: string, block?: } logger.error("Request could not be verified.", {method: payload.method, params: JSON.stringify(payload.params)}); - return getErrorResponseForUnverifiedRequest(payload, "eth_getBalance request can not be verified."); + return getErrorResponseForRequestWithFailedVerification(payload, getVerificationFailedMessage("eth_getBalance")); }; diff --git a/packages/prover/src/verified_requests/eth_getBlockByHash.ts b/packages/prover/src/verified_requests/eth_getBlockByHash.ts index 0c27a81729ad..cb5fa1711c0f 100644 --- a/packages/prover/src/verified_requests/eth_getBlockByHash.ts +++ b/packages/prover/src/verified_requests/eth_getBlockByHash.ts @@ -1,7 +1,11 @@ import {ELVerifiedRequestHandler} from "../interfaces.js"; import {ELBlock} from "../types.js"; import {verifyBlock} from "../utils/verification.js"; -import {getErrorResponseForUnverifiedRequest, getResponseForRequest} from "../utils/json_rpc.js"; +import { + getErrorResponseForRequestWithFailedVerification, + getResponseForRequest, + getVerificationFailedMessage, +} from "../utils/json_rpc.js"; // eslint-disable-next-line @typescript-eslint/naming-convention export const eth_getBlockByHash: ELVerifiedRequestHandler<[block: string, hydrated: boolean], ELBlock> = async ({ @@ -17,5 +21,5 @@ export const eth_getBlockByHash: ELVerifiedRequestHandler<[block: string, hydrat } logger.error("Request could not be verified.", {method: payload.method, params: JSON.stringify(payload.params)}); - return getErrorResponseForUnverifiedRequest(payload, "eth_getBlockByHash request can not be verified."); + return getErrorResponseForRequestWithFailedVerification(payload, getVerificationFailedMessage("eth_getBlockByHash")); }; diff --git a/packages/prover/src/verified_requests/eth_getBlockByNumber.ts b/packages/prover/src/verified_requests/eth_getBlockByNumber.ts index 64c00410945a..a08703881cc0 100644 --- a/packages/prover/src/verified_requests/eth_getBlockByNumber.ts +++ b/packages/prover/src/verified_requests/eth_getBlockByNumber.ts @@ -1,7 +1,11 @@ import {ELVerifiedRequestHandler} from "../interfaces.js"; import {ELBlock} from "../types.js"; import {verifyBlock} from "../utils/verification.js"; -import {getErrorResponseForUnverifiedRequest, getResponseForRequest} from "../utils/json_rpc.js"; +import { + getErrorResponseForRequestWithFailedVerification, + getResponseForRequest, + getVerificationFailedMessage, +} from "../utils/json_rpc.js"; // eslint-disable-next-line @typescript-eslint/naming-convention export const eth_getBlockByNumber: ELVerifiedRequestHandler< @@ -15,5 +19,8 @@ export const eth_getBlockByNumber: ELVerifiedRequestHandler< } logger.error("Request could not be verified.", {method: payload.method, params: JSON.stringify(payload.params)}); - return getErrorResponseForUnverifiedRequest(payload, "eth_getBlockByNumber request can not be verified."); + return getErrorResponseForRequestWithFailedVerification( + payload, + getVerificationFailedMessage("eth_getBlockByNumber") + ); }; diff --git a/packages/prover/src/verified_requests/eth_getCode.ts b/packages/prover/src/verified_requests/eth_getCode.ts index 9f64ae3627f3..9cb3362c4e50 100644 --- a/packages/prover/src/verified_requests/eth_getCode.ts +++ b/packages/prover/src/verified_requests/eth_getCode.ts @@ -1,6 +1,10 @@ import {ELVerifiedRequestHandler} from "../interfaces.js"; import {verifyAccount, verifyCode} from "../utils/verification.js"; -import {getErrorResponseForUnverifiedRequest, getResponseForRequest} from "../utils/json_rpc.js"; +import { + getErrorResponseForRequestWithFailedVerification, + getResponseForRequest, + getVerificationFailedMessage, +} from "../utils/json_rpc.js"; // eslint-disable-next-line @typescript-eslint/naming-convention export const eth_getCode: ELVerifiedRequestHandler<[address: string, block?: number | string], string> = async ({ @@ -23,7 +27,10 @@ export const eth_getCode: ELVerifiedRequestHandler<[address: string, block?: num if (!accountProof.valid) { logger.error("Request could not be verified.", {method: payload.method, params: JSON.stringify(payload.params)}); - return getErrorResponseForUnverifiedRequest(payload, "account for eth_getCode request can not be verified."); + return getErrorResponseForRequestWithFailedVerification( + payload, + "account for eth_getCode request can not be verified." + ); } const codeProof = await verifyCode({ @@ -40,5 +47,5 @@ export const eth_getCode: ELVerifiedRequestHandler<[address: string, block?: num } logger.error("Request could not be verified.", {method: payload.method, params: JSON.stringify(payload.params)}); - return getErrorResponseForUnverifiedRequest(payload, "eth_getCode request can not be verified."); + return getErrorResponseForRequestWithFailedVerification(payload, getVerificationFailedMessage("eth_getCode")); }; diff --git a/packages/prover/src/verified_requests/eth_getTransactionCount.ts b/packages/prover/src/verified_requests/eth_getTransactionCount.ts index 8a2de9c78179..4cafd9b2b271 100644 --- a/packages/prover/src/verified_requests/eth_getTransactionCount.ts +++ b/packages/prover/src/verified_requests/eth_getTransactionCount.ts @@ -1,6 +1,10 @@ import {ELVerifiedRequestHandler} from "../interfaces.js"; import {verifyAccount} from "../utils/verification.js"; -import {getResponseForRequest, getErrorResponseForUnverifiedRequest} from "../utils/json_rpc.js"; +import { + getResponseForRequest, + getErrorResponseForRequestWithFailedVerification, + getVerificationFailedMessage, +} from "../utils/json_rpc.js"; // eslint-disable-next-line @typescript-eslint/naming-convention export const eth_getTransactionCount: ELVerifiedRequestHandler< @@ -17,5 +21,8 @@ export const eth_getTransactionCount: ELVerifiedRequestHandler< } logger.error("Request could not be verified.", {method: payload.method, params: JSON.stringify(payload.params)}); - return getErrorResponseForUnverifiedRequest(payload, "eth_getTransactionCount request can not be verified."); + return getErrorResponseForRequestWithFailedVerification( + payload, + getVerificationFailedMessage("eth_getTransactionCount") + ); }; diff --git a/packages/prover/src/web3_proxy.ts b/packages/prover/src/web3_proxy.ts index 508aed45b86c..6aa44b314953 100644 --- a/packages/prover/src/web3_proxy.ts +++ b/packages/prover/src/web3_proxy.ts @@ -1,4 +1,5 @@ import http from "node:http"; +import https from "node:https"; import url from "node:url"; import httpProxy from "http-proxy"; import {getNodeLogger} from "@lodestar/logger/node"; @@ -78,7 +79,7 @@ export function createVerifiedExecutionProxy(opts: VerifiedProxyOptions): { const proxy = httpProxy.createProxy({ target: executionRpcUrl, ws: executionRpcUrl.startsWith("ws"), - agent: http.globalAgent, + agent: executionRpcUrl.startsWith("https") ? https.globalAgent : http.globalAgent, xfwd: true, ignorePath: true, changeOrigin: true, diff --git a/packages/prover/test/e2e/web3_batch_request.test.ts b/packages/prover/test/e2e/web3_batch_request.test.ts index 472c04274764..f7167a8054f3 100644 --- a/packages/prover/test/e2e/web3_batch_request.test.ts +++ b/packages/prover/test/e2e/web3_batch_request.test.ts @@ -4,6 +4,7 @@ import Web3 from "web3"; import {LCTransport} from "../../src/interfaces.js"; import {createVerifiedExecutionProvider} from "../../src/web3_provider.js"; import {rpcUrl, beaconUrl, config} from "../utils/e2e_env.js"; +import {getVerificationFailedMessage} from "../../src/utils/json_rpc.js"; describe("web3_batch_requests", function () { // Give some margin to sync light client @@ -94,7 +95,7 @@ describe("web3_batch_requests", function () { batch.execute(); await expect(successRequest).to.be.fulfilled; - await expect(errorRequest).to.be.rejectedWith("eth_getBlockByHash request can not be verified"); + await expect(errorRequest).to.be.rejectedWith(getVerificationFailedMessage("eth_getBlockByHash")); }); }); }); diff --git a/packages/prover/test/mocks/request_handler.ts b/packages/prover/test/mocks/request_handler.ts index ba71954200a4..9f65eee00363 100644 --- a/packages/prover/test/mocks/request_handler.ts +++ b/packages/prover/test/mocks/request_handler.ts @@ -117,6 +117,7 @@ export function generateReqHandlerOptionsMock( rpc: { request: sinon.stub(), batchRequest: sinon.stub(), + getRequestId: () => (Math.random() * 10000).toFixed(0), }, }; diff --git a/packages/prover/test/unit/verified_requests/eth_call.test.ts b/packages/prover/test/unit/verified_requests/eth_call.test.ts index 1c60879c8591..996ce0849e8a 100644 --- a/packages/prover/test/unit/verified_requests/eth_call.test.ts +++ b/packages/prover/test/unit/verified_requests/eth_call.test.ts @@ -3,11 +3,12 @@ import deepmerge from "deepmerge"; import {createForkConfig} from "@lodestar/config"; import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; import {ELTransaction} from "../../../lib/types.js"; -import {UNVERIFIED_RESPONSE_CODE} from "../../../src/constants.js"; +import {VERIFICATION_FAILED_RESPONSE_CODE} from "../../../src/constants.js"; import {eth_call} from "../../../src/verified_requests/eth_call.js"; import ethCallCase1 from "../../fixtures/mainnet/eth_call.json" assert {type: "json"}; import {generateReqHandlerOptionsMock} from "../../mocks/request_handler.js"; import {JsonRpcRequest, JsonRpcResponseWithResultPayload} from "../../../src/types.js"; +import {getVerificationFailedMessage} from "../../../src/utils/json_rpc.js"; const testCases = [ethCallCase1]; @@ -60,7 +61,7 @@ describe("verified_requests / eth_call", () => { expect(response).to.eql({ jsonrpc: "2.0", id: testCase.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_call request can not be verified."}, + error: {code: VERIFICATION_FAILED_RESPONSE_CODE, message: getVerificationFailedMessage("eth_call")}, }); }); }); diff --git a/packages/prover/test/unit/verified_requests/eth_estimateGas.test.ts b/packages/prover/test/unit/verified_requests/eth_estimateGas.test.ts index dd87e8e466b6..ae7df1e8b7fb 100644 --- a/packages/prover/test/unit/verified_requests/eth_estimateGas.test.ts +++ b/packages/prover/test/unit/verified_requests/eth_estimateGas.test.ts @@ -3,12 +3,13 @@ import deepmerge from "deepmerge"; import {createForkConfig} from "@lodestar/config"; import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; import {ELTransaction} from "../../../lib/types.js"; -import {UNVERIFIED_RESPONSE_CODE} from "../../../src/constants.js"; +import {VERIFICATION_FAILED_RESPONSE_CODE} from "../../../src/constants.js"; import {eth_estimateGas} from "../../../src/verified_requests/eth_estimateGas.js"; import ethEstimateGasCase1 from "../../fixtures/mainnet/eth_estimateGas_simple_transfer.json" assert {type: "json"}; import ethEstimateGasCase2 from "../../fixtures/mainnet/eth_estimateGas_contract_call.json" assert {type: "json"}; import {TestFixture, generateReqHandlerOptionsMock} from "../../mocks/request_handler.js"; import {JsonRpcRequest, JsonRpcResponseWithResultPayload} from "../../../src/types.js"; +import {getVerificationFailedMessage} from "../../../src/utils/json_rpc.js"; const testCases = [ethEstimateGasCase1, ethEstimateGasCase2] as TestFixture[]; @@ -62,7 +63,7 @@ describe("verified_requests / eth_estimateGas", () => { expect(response).to.eql({ jsonrpc: "2.0", id: testCase.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_estimateGas request can not be verified."}, + error: {code: VERIFICATION_FAILED_RESPONSE_CODE, message: getVerificationFailedMessage("eth_estimateGas")}, }); }); }); diff --git a/packages/prover/test/unit/verified_requests/eth_getBalance.test.ts b/packages/prover/test/unit/verified_requests/eth_getBalance.test.ts index cf8dd4e85c00..46b4edf77f16 100644 --- a/packages/prover/test/unit/verified_requests/eth_getBalance.test.ts +++ b/packages/prover/test/unit/verified_requests/eth_getBalance.test.ts @@ -2,11 +2,12 @@ import {expect} from "chai"; import deepmerge from "deepmerge"; import {createForkConfig} from "@lodestar/config"; import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; -import {UNVERIFIED_RESPONSE_CODE} from "../../../src/constants.js"; +import {VERIFICATION_FAILED_RESPONSE_CODE} from "../../../src/constants.js"; import {eth_getBalance} from "../../../src/verified_requests/eth_getBalance.js"; import eth_getBalance_eoa from "../../fixtures/sepolia/eth_getBalance_eoa.json" assert {type: "json"}; import eth_getBalance_contract from "../../fixtures/sepolia/eth_getBalance_contract.json" assert {type: "json"}; import {generateReqHandlerOptionsMock} from "../../mocks/request_handler.js"; +import {getVerificationFailedMessage} from "../../../src/utils/json_rpc.js"; const testCases = [eth_getBalance_eoa, eth_getBalance_contract]; @@ -46,7 +47,7 @@ describe("verified_requests / eth_getBalance", () => { expect(response).to.eql({ jsonrpc: "2.0", id: data.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_getBalance request can not be verified."}, + error: {code: VERIFICATION_FAILED_RESPONSE_CODE, message: getVerificationFailedMessage("eth_getBalance")}, }); }); }); diff --git a/packages/prover/test/unit/verified_requests/eth_getBlockByHash.test.ts b/packages/prover/test/unit/verified_requests/eth_getBlockByHash.test.ts index 71c584a74f1d..a55f94903216 100644 --- a/packages/prover/test/unit/verified_requests/eth_getBlockByHash.test.ts +++ b/packages/prover/test/unit/verified_requests/eth_getBlockByHash.test.ts @@ -2,12 +2,13 @@ import {expect} from "chai"; import deepmerge from "deepmerge"; import {createForkConfig} from "@lodestar/config"; import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; -import {UNVERIFIED_RESPONSE_CODE} from "../../../src/constants.js"; +import {VERIFICATION_FAILED_RESPONSE_CODE} from "../../../src/constants.js"; import {eth_getBlockByHash} from "../../../src/verified_requests/eth_getBlockByHash.js"; import eth_getBlock_with_contractCreation from "../../fixtures/sepolia/eth_getBlock_with_contractCreation.json" assert {type: "json"}; import eth_getBlock_with_no_accessList from "../../fixtures/sepolia/eth_getBlock_with_no_accessList.json" assert {type: "json"}; import {TestFixture, generateReqHandlerOptionsMock} from "../../mocks/request_handler.js"; import {ELBlock} from "../../../src/types.js"; +import {getVerificationFailedMessage} from "../../../src/utils/json_rpc.js"; const testCases = [eth_getBlock_with_no_accessList, eth_getBlock_with_contractCreation] as [ TestFixture, @@ -51,7 +52,7 @@ describe("verified_requests / eth_getBlockByHash", () => { expect(response).to.eql({ jsonrpc: "2.0", id: testCase.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_getBlockByHash request can not be verified."}, + error: {code: VERIFICATION_FAILED_RESPONSE_CODE, message: getVerificationFailedMessage("eth_getBlockByHash")}, }); }); @@ -74,7 +75,7 @@ describe("verified_requests / eth_getBlockByHash", () => { expect(response).to.eql({ jsonrpc: "2.0", id: testCase.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_getBlockByHash request can not be verified."}, + error: {code: VERIFICATION_FAILED_RESPONSE_CODE, message: getVerificationFailedMessage("eth_getBlockByHash")}, }); }); @@ -97,7 +98,7 @@ describe("verified_requests / eth_getBlockByHash", () => { expect(response).to.eql({ jsonrpc: "2.0", id: testCase.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_getBlockByHash request can not be verified."}, + error: {code: VERIFICATION_FAILED_RESPONSE_CODE, message: getVerificationFailedMessage("eth_getBlockByHash")}, }); }); }); diff --git a/packages/prover/test/unit/verified_requests/eth_getBlockByNumber.test.ts b/packages/prover/test/unit/verified_requests/eth_getBlockByNumber.test.ts index a4f1f9d7f869..506a115eba4d 100644 --- a/packages/prover/test/unit/verified_requests/eth_getBlockByNumber.test.ts +++ b/packages/prover/test/unit/verified_requests/eth_getBlockByNumber.test.ts @@ -2,12 +2,13 @@ import {expect} from "chai"; import deepmerge from "deepmerge"; import {createForkConfig} from "@lodestar/config"; import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; -import {UNVERIFIED_RESPONSE_CODE} from "../../../src/constants.js"; +import {VERIFICATION_FAILED_RESPONSE_CODE} from "../../../src/constants.js"; import {ELBlock} from "../../../src/types.js"; import {eth_getBlockByNumber} from "../../../src/verified_requests/eth_getBlockByNumber.js"; import eth_getBlock_with_contractCreation from "../../fixtures/sepolia/eth_getBlock_with_contractCreation.json" assert {type: "json"}; import eth_getBlock_with_no_accessList from "../../fixtures/sepolia/eth_getBlock_with_no_accessList.json" assert {type: "json"}; import {TestFixture, generateReqHandlerOptionsMock} from "../../mocks/request_handler.js"; +import {getVerificationFailedMessage} from "../../../src/utils/json_rpc.js"; const testCases = [eth_getBlock_with_no_accessList, eth_getBlock_with_contractCreation] as [ TestFixture, @@ -51,7 +52,10 @@ describe("verified_requests / eth_getBlockByNumber", () => { expect(response).to.eql({ jsonrpc: "2.0", id: testCase.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_getBlockByNumber request can not be verified."}, + error: { + code: VERIFICATION_FAILED_RESPONSE_CODE, + message: getVerificationFailedMessage("eth_getBlockByNumber"), + }, }); }); @@ -74,7 +78,10 @@ describe("verified_requests / eth_getBlockByNumber", () => { expect(response).to.eql({ jsonrpc: "2.0", id: testCase.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_getBlockByNumber request can not be verified."}, + error: { + code: VERIFICATION_FAILED_RESPONSE_CODE, + message: getVerificationFailedMessage("eth_getBlockByNumber"), + }, }); }); @@ -100,7 +107,10 @@ describe("verified_requests / eth_getBlockByNumber", () => { expect(response).to.eql({ jsonrpc: "2.0", id: testCase.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_getBlockByNumber request can not be verified."}, + error: { + code: VERIFICATION_FAILED_RESPONSE_CODE, + message: getVerificationFailedMessage("eth_getBlockByNumber"), + }, }); }); }); diff --git a/packages/prover/test/unit/verified_requests/eth_getCode.test.ts b/packages/prover/test/unit/verified_requests/eth_getCode.test.ts index 584457dff781..c88f58a1cd2b 100644 --- a/packages/prover/test/unit/verified_requests/eth_getCode.test.ts +++ b/packages/prover/test/unit/verified_requests/eth_getCode.test.ts @@ -2,10 +2,11 @@ import {expect} from "chai"; import deepmerge from "deepmerge"; import {createForkConfig} from "@lodestar/config"; import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; -import {UNVERIFIED_RESPONSE_CODE} from "../../../src/constants.js"; +import {VERIFICATION_FAILED_RESPONSE_CODE} from "../../../src/constants.js"; import {eth_getCode} from "../../../src/verified_requests/eth_getCode.js"; import ethGetCodeCase1 from "../../fixtures/sepolia/eth_getCode.json" assert {type: "json"}; import {generateReqHandlerOptionsMock} from "../../mocks/request_handler.js"; +import {getVerificationFailedMessage} from "../../../src/utils/json_rpc.js"; const testCases = [ethGetCodeCase1]; @@ -44,7 +45,7 @@ describe("verified_requests / eth_getCode", () => { expect(response).to.eql({ jsonrpc: "2.0", id: testCase.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_getCode request can not be verified."}, + error: {code: VERIFICATION_FAILED_RESPONSE_CODE, message: getVerificationFailedMessage("eth_getCode")}, }); }); }); diff --git a/packages/prover/test/unit/verified_requests/eth_getTransactionCount.test.ts b/packages/prover/test/unit/verified_requests/eth_getTransactionCount.test.ts index b6fd3dcecaad..03d491dc5b0c 100644 --- a/packages/prover/test/unit/verified_requests/eth_getTransactionCount.test.ts +++ b/packages/prover/test/unit/verified_requests/eth_getTransactionCount.test.ts @@ -2,10 +2,11 @@ import {expect} from "chai"; import deepmerge from "deepmerge"; import {createForkConfig} from "@lodestar/config"; import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; -import {UNVERIFIED_RESPONSE_CODE} from "../../../src/constants.js"; +import {VERIFICATION_FAILED_RESPONSE_CODE} from "../../../src/constants.js"; import {eth_getTransactionCount} from "../../../src/verified_requests/eth_getTransactionCount.js"; import getTransactionCountCase1 from "../../fixtures/sepolia/eth_getTransactionCount.json" assert {type: "json"}; import {generateReqHandlerOptionsMock} from "../../mocks/request_handler.js"; +import {getVerificationFailedMessage} from "../../../src/utils/json_rpc.js"; const testCases = [getTransactionCountCase1]; @@ -46,7 +47,10 @@ describe("verified_requests / eth_getTransactionCount", () => { expect(response).to.eql({ jsonrpc: "2.0", id: testCase.request.id, - error: {code: UNVERIFIED_RESPONSE_CODE, message: "eth_getTransactionCount request can not be verified."}, + error: { + code: VERIFICATION_FAILED_RESPONSE_CODE, + message: getVerificationFailedMessage("eth_getTransactionCount"), + }, }); }); }); diff --git a/packages/prover/test/unit/web3_provider.test.ts b/packages/prover/test/unit/web3_provider.test.ts index 290dfce72f3f..e29188503b96 100644 --- a/packages/prover/test/unit/web3_provider.test.ts +++ b/packages/prover/test/unit/web3_provider.test.ts @@ -2,9 +2,7 @@ import {expect} from "chai"; import Web3 from "web3"; import {ethers} from "ethers"; import sinon from "sinon"; -import {LCTransport} from "../../src/interfaces.js"; -import {ProofProvider} from "../../src/proof_provider/proof_provider.js"; -import {createVerifiedExecutionProvider} from "../../src/web3_provider.js"; +import {createVerifiedExecutionProvider, ProofProvider, LCTransport} from "@lodestar/prover/browser"; import {ELRpc} from "../../src/utils/rpc.js"; describe("web3_provider", () => { diff --git a/scripts/assert_exports.mjs b/scripts/assert_exports.mjs index 1f52c909aad5..5110fbc79658 100644 --- a/scripts/assert_exports.mjs +++ b/scripts/assert_exports.mjs @@ -5,22 +5,13 @@ import path from "node:path"; // This script ensure that the referenced files exist const pkgsDirpath = path.resolve("./packages"); -const exportPaths = []; - -for (const pkgDirname of fs.readdirSync(pkgsDirpath)) { - const pkgDirpath = path.join(pkgsDirpath, pkgDirname); - const packageJSONPath = path.join(pkgDirpath, "package.json"); - if (!fs.existsSync(packageJSONPath)) { - throw Error(`No package.json found in ${pkgDirpath}`); - } - - const packageJSON = JSON.parse(fs.readFileSync(packageJSONPath, "utf8")); +function getExportPaths(pkgDirPath, pkgExports) { // { // "exports": "./lib/index.js", // } - if (typeof packageJSON.exports === "string") { - exportPaths.push(path.join(pkgDirpath, packageJSON.exports)); + if (typeof pkgExports === "string") { + return [pkgExports]; } // { @@ -29,18 +20,30 @@ for (const pkgDirname of fs.readdirSync(pkgsDirpath)) { // "import": "./lib/index.js" // }, // } - else if (typeof packageJSON.exports === "object") { - for (const [exportPath, exportObj] of Object.entries(packageJSON.exports)) { - if (!exportObj.import) { - throw Error(`package.json ${packageJSONPath} export ${exportPath} has not import`); - } - - exportPaths.push(path.join(pkgDirpath, exportObj.import)); + const exportPaths = []; + for (const [exportPath, nestedExportObj] of Object.entries(pkgExports)) { + if (typeof nestedExportObj === "object") { + exportPaths.push(...getExportPaths(pkgDirPath, nestedExportObj)); + } else if (typeof nestedExportObj === "string") { + exportPaths.push(nestedExportObj); } } + + return exportPaths; } -const missingExportPaths = exportPaths.filter((exportPath) => !fs.existsSync(exportPath)); -if (missingExportPaths.length > 0) { - throw Error(`export paths file(s) not found\n${missingExportPaths.join("\n")}`); +for (const pkgDirname of fs.readdirSync(pkgsDirpath)) { + const pkgDirpath = path.join(pkgsDirpath, pkgDirname); + const packageJSONPath = path.join(pkgDirpath, "package.json"); + if (!fs.existsSync(packageJSONPath)) { + throw Error(`No package.json found in ${pkgDirpath}`); + } + + const packageJSON = JSON.parse(fs.readFileSync(packageJSONPath, "utf8")); + const exportPaths = getExportPaths(pkgDirpath, packageJSON.exports); + const missingExportPaths = exportPaths.filter((exportPath) => !fs.existsSync(path.join(pkgDirpath, exportPath))); + + if (missingExportPaths.length > 0) { + throw Error(`export paths file(s) not found\n${missingExportPaths.join("\n")}`); + } } From 4720a5bf75036b180bc7b904d4cf3741a2a4c098 Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Mon, 24 Jul 2023 12:55:55 -0400 Subject: [PATCH 83/96] test(db): add check for gdu vs du (#5788) tes(db)t: add check for gdu vs du --- packages/db/test/unit/controller/level.test.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/db/test/unit/controller/level.test.ts b/packages/db/test/unit/controller/level.test.ts index 337274b5eb53..38d8274ebb22 100644 --- a/packages/db/test/unit/controller/level.test.ts +++ b/packages/db/test/unit/controller/level.test.ts @@ -1,4 +1,5 @@ import {execSync} from "node:child_process"; +import os from "node:os"; import {expect} from "chai"; import leveldown from "leveldown"; import all from "it-all"; @@ -135,9 +136,23 @@ describe("LevelDB controller", () => { expect(approxSize).gt(0, "approximateSize return not > 0"); }); + function getDuCommand(): string { + if (os.platform() === "darwin") { + try { + const res = execSync("gdu --help", {encoding: "utf8"}); + if (res?.startsWith("Usage: gdu ")) { + return "gdu"; + } + } catch { + /* no-op */ + } + } + return "du"; + } + function getDbSize(): number { // 116 ./.__testdb - const res = execSync(`du -bs ${dbLocation}`, {encoding: "utf8"}); + const res = execSync(`${getDuCommand()} -bs ${dbLocation}`, {encoding: "utf8"}); const match = res.match(/^(\d+)/); if (!match) throw Error(`Unknown du response \n${res}`); return parseInt(match[1]); From 2f28718a17416c68d401c808fb9d737571fd2a87 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 24 Jul 2023 18:57:15 +0200 Subject: [PATCH 84/96] refactor: update distributed aggregation selection error logs (#5780) --- packages/validator/src/services/attestation.ts | 6 ++---- packages/validator/src/services/syncCommittee.ts | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index b5bd851760fd..f954a9d2d0d5 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -322,9 +322,7 @@ export class AttestationService { this.logger.debug("Submitting partial beacon committee selection proofs", {slot, count: partialSelections.length}); const res = await Promise.race([ - this.api.validator - .submitBeaconCommitteeSelections(partialSelections) - .catch((e) => this.logger.error("Error on submitBeaconCommitteeSelections", {slot}, e)), + this.api.validator.submitBeaconCommitteeSelections(partialSelections), // Exit attestation aggregation flow if there is no response after 1/3 of slot as // beacon node would likely not have enough time to prepare an aggregate attestation. // Note that the aggregations flow is not explicitly exited but rather will be skipped @@ -334,7 +332,7 @@ export class AttestationService { ]); if (!res) { - throw new Error("submitBeaconCommitteeSelections did not resolve after 1/3 of slot"); + throw new Error("Failed to receive combined selection proofs before 1/3 of slot"); } ApiError.assert(res, "Error receiving combined selection proofs"); diff --git a/packages/validator/src/services/syncCommittee.ts b/packages/validator/src/services/syncCommittee.ts index adbda3231697..9f104e0d7d4b 100644 --- a/packages/validator/src/services/syncCommittee.ts +++ b/packages/validator/src/services/syncCommittee.ts @@ -261,9 +261,7 @@ export class SyncCommitteeService { this.logger.debug("Submitting partial sync committee selection proofs", {slot, count: partialSelections.length}); const res = await Promise.race([ - this.api.validator - .submitSyncCommitteeSelections(partialSelections) - .catch((e) => this.logger.error("Error on submitSyncCommitteeSelections", {slot}, e)), + this.api.validator.submitSyncCommitteeSelections(partialSelections), // Exit sync committee contributions flow if there is no response after 2/3 of slot. // This is in contrast to attestations aggregations flow which is already exited at 1/3 of the slot // because for sync committee is not required to resubscribe to subnets as beacon node will assume @@ -275,7 +273,7 @@ export class SyncCommitteeService { ]); if (!res) { - throw new Error("submitSyncCommitteeSelections did not resolve after 2/3 of slot"); + throw new Error("Failed to receive combined selection proofs before 2/3 of slot"); } ApiError.assert(res, "Error receiving combined selection proofs"); From ae9f572c8e88472b28803733d353f492ff74bfc4 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 24 Jul 2023 18:59:35 +0200 Subject: [PATCH 85/96] fix: gracefully terminate connections when closing http server (#5786) * Gracefully terminate connections when closing http server * Log unexpected errors when shutting down server * Fix code references in comments * More gracefully close eventstream api * Misc updates * Close idle connections on server * Update ts-expect-error comments --- .../beacon-node/src/api/rest/activeSockets.ts | 74 ++++++++++++++++--- packages/beacon-node/src/api/rest/base.ts | 25 ++----- .../beacon-node/src/metrics/server/http.ts | 8 +- packages/utils/src/index.ts | 1 + packages/utils/src/waitFor.ts | 53 +++++++++++++ packages/utils/test/unit/waitFor.test.ts | 37 ++++++++++ 6 files changed, 166 insertions(+), 32 deletions(-) create mode 100644 packages/utils/src/waitFor.ts create mode 100644 packages/utils/test/unit/waitFor.test.ts diff --git a/packages/beacon-node/src/api/rest/activeSockets.ts b/packages/beacon-node/src/api/rest/activeSockets.ts index b98db9b0fb9b..ba8a35c80119 100644 --- a/packages/beacon-node/src/api/rest/activeSockets.ts +++ b/packages/beacon-node/src/api/rest/activeSockets.ts @@ -1,5 +1,6 @@ import http, {Server} from "node:http"; import {Socket} from "node:net"; +import {waitFor} from "@lodestar/utils"; import {IGauge} from "../../metrics/index.js"; export type SocketMetrics = { @@ -8,17 +9,23 @@ export type SocketMetrics = { socketsBytesWritten: IGauge; }; +// Use relatively short timeout to speed up shutdown +const GRACEFUL_TERMINATION_TIMEOUT = 1_000; + /** - * From https://github.com/nodejs/node/blob/57bd715d527aba8dae56b975056961b0e429e91e/lib/_http_client.js#L363-L413 - * But exposes the count of sockets, and does not have a graceful period + * From https://github.com/gajus/http-terminator/blob/aabca4751552e983f8a59ba896b7fb58ce3b4087/src/factories/createInternalHttpTerminator.ts#L24-L61 + * But only handles HTTP sockets, exposes the count of sockets as metrics */ export class HttpActiveSocketsTracker { private sockets = new Set(); - private terminated = false; + private terminating = false; - constructor(server: Server, metrics: SocketMetrics | null) { + constructor( + private readonly server: Server, + metrics: SocketMetrics | null + ) { server.on("connection", (socket) => { - if (this.terminated) { + if (this.terminating) { socket.destroy(Error("Closing")); } else { this.sockets.add(socket); @@ -39,18 +46,65 @@ export class HttpActiveSocketsTracker { } } - destroyAll(): void { - this.terminated = true; + /** + * Wait for all connections to drain, forcefully terminate any open connections after timeout + * + * From https://github.com/gajus/http-terminator/blob/aabca4751552e983f8a59ba896b7fb58ce3b4087/src/factories/createInternalHttpTerminator.ts#L78-L165 + * But only handles HTTP sockets and does not close server, immediately closes eventstream API connections + */ + async terminate(): Promise { + if (this.terminating) return; + this.terminating = true; + + // Can speed up shutdown by a few milliseconds + this.server.closeIdleConnections(); + + // Inform new incoming requests on keep-alive connections that + // the connection will be closed after the current response + this.server.on("request", (_req, res) => { + if (!res.headersSent) { + res.setHeader("Connection", "close"); + } + }); for (const socket of this.sockets) { // This is the HTTP CONNECT request socket. - // @ts-expect-error Unclear if I am using wrong type or how else this should be handled. + // @ts-expect-error HTTP sockets have reference to server if (!(socket.server instanceof http.Server)) { continue; } - socket.destroy(Error("Closing")); - this.sockets.delete(socket); + // @ts-expect-error Internal property but only way to access response of socket + const res = socket._httpMessage as http.ServerResponse | undefined; + + if (res == null) { + // Immediately destroy sockets without an attached HTTP request + this.destroySocket(socket); + } else if (res.getHeader("Content-Type") === "text/event-stream") { + // eventstream API will never stop and must be forcefully closed + socket.end(); + } else if (!res.headersSent) { + // Inform existing keep-alive connections that they will be closed after the current response + res.setHeader("Connection", "close"); + } + } + + // Wait for all connections to drain, forcefully terminate after timeout + try { + await waitFor(() => this.sockets.size === 0, { + timeout: GRACEFUL_TERMINATION_TIMEOUT, + }); + } catch { + // Ignore timeout error + } finally { + for (const socket of this.sockets) { + this.destroySocket(socket); + } } } + + private destroySocket(socket: Socket): void { + socket.destroy(Error("Closing")); + this.sockets.delete(socket); + } } diff --git a/packages/beacon-node/src/api/rest/base.ts b/packages/beacon-node/src/api/rest/base.ts index fda16fffc8ce..c9c6e4bfb0ef 100644 --- a/packages/beacon-node/src/api/rest/base.ts +++ b/packages/beacon-node/src/api/rest/base.ts @@ -29,11 +29,6 @@ export type RestApiServerMetrics = SocketMetrics & { errors: IGauge<"operationId">; }; -enum Status { - Listening = "listening", - Closed = "closed", -} - /** * REST API powered by `fastify` server. */ @@ -42,8 +37,6 @@ export class RestApiServer { protected readonly logger: Logger; private readonly activeSockets: HttpActiveSocketsTracker; - private status = Status.Closed; - constructor( private readonly opts: RestApiServerOpts, modules: RestApiServerModules @@ -106,8 +99,7 @@ export class RestApiServer { server.addHook("onError", async (req, _res, err) => { // Don't log ErrorAborted errors, they happen on node shutdown and are not useful // Don't log NodeISSyncing errors, they happen very frequently while syncing and the validator polls duties - // Don't log eventstream aborted errors if server instance is being closed on node shutdown - if (err instanceof ErrorAborted || err instanceof NodeIsSyncing || this.status === Status.Closed) return; + if (err instanceof ErrorAborted || err instanceof NodeIsSyncing) return; const {operationId} = req.routeConfig as RouteConfig; @@ -127,9 +119,6 @@ export class RestApiServer { * Start the REST API server. */ async listen(): Promise { - if (this.status === Status.Listening) return; - this.status = Status.Listening; - try { const host = this.opts.address; const address = await this.server.listen({port: this.opts.port, host}); @@ -139,7 +128,6 @@ export class RestApiServer { } } catch (e) { this.logger.error("Error starting REST api server", this.opts, e as Error); - this.status = Status.Closed; throw e; } } @@ -148,17 +136,16 @@ export class RestApiServer { * Close the server instance and terminate all existing connections. */ async close(): Promise { - if (this.status === Status.Closed) return; - this.status = Status.Closed; - // In NodeJS land calling close() only causes new connections to be rejected. // Existing connections can prevent .close() from resolving for potentially forever. - // In Lodestar case when the BeaconNode wants to close we will just abruptly terminate - // all existing connections for a fast shutdown. + // In Lodestar case when the BeaconNode wants to close we will attempt to gracefully + // close all existing connections but forcefully terminate after timeout for a fast shutdown. // Inspired by https://github.com/gajus/http-terminator/ - this.activeSockets.destroyAll(); + await this.activeSockets.terminate(); await this.server.close(); + + this.logger.debug("REST API server closed"); } /** For child classes to override */ diff --git a/packages/beacon-node/src/metrics/server/http.ts b/packages/beacon-node/src/metrics/server/http.ts index ba17b743d112..b699471e07d5 100644 --- a/packages/beacon-node/src/metrics/server/http.ts +++ b/packages/beacon-node/src/metrics/server/http.ts @@ -90,10 +90,10 @@ export async function getHttpMetricsServer( async close(): Promise { // In NodeJS land calling close() only causes new connections to be rejected. // Existing connections can prevent .close() from resolving for potentially forever. - // In Lodestar case when the BeaconNode wants to close we will just abruptly terminate - // all existing connections for a fast shutdown. + // In Lodestar case when the BeaconNode wants to close we will attempt to gracefully + // close all existing connections but forcefully terminate after timeout for a fast shutdown. // Inspired by https://github.com/gajus/http-terminator/ - activeSockets.destroyAll(); + await activeSockets.terminate(); await new Promise((resolve, reject) => { server.close((err) => { @@ -101,6 +101,8 @@ export async function getHttpMetricsServer( else resolve(); }); }); + + logger.debug("Metrics HTTP server closed"); }, }; } diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index bcb0bf27109f..a09c615fbf2b 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -16,3 +16,4 @@ export * from "./timeout.js"; export {RecursivePartial, bnToNum} from "./types.js"; export * from "./verifyMerkleBranch.js"; export * from "./promise.js"; +export * from "./waitFor.js"; diff --git a/packages/utils/src/waitFor.ts b/packages/utils/src/waitFor.ts new file mode 100644 index 000000000000..91206267e6ca --- /dev/null +++ b/packages/utils/src/waitFor.ts @@ -0,0 +1,53 @@ +import {ErrorAborted, TimeoutError} from "./errors.js"; + +export type WaitForOpts = { + /** Time in milliseconds between checking condition */ + interval?: number; + /** Time in milliseconds to wait before throwing TimeoutError */ + timeout?: number; + /** Abort signal to stop waiting for condition by throwing ErrorAborted */ + signal?: AbortSignal; +}; + +/** + * Wait for a condition to be true + */ +export function waitFor(condition: () => boolean, opts: WaitForOpts = {}): Promise { + return new Promise((resolve, reject) => { + const {interval = 10, timeout = Infinity, signal} = opts; + + if (signal?.aborted) { + return reject(new ErrorAborted()); + } + + if (condition()) { + return resolve(); + } + + let onDone: () => void = () => {}; + + const timeoutId = setTimeout(() => { + onDone(); + reject(new TimeoutError()); + }, timeout); + + const intervalId = setInterval(() => { + if (condition()) { + onDone(); + resolve(); + } + }, interval); + + const onAbort = (): void => { + onDone(); + reject(new ErrorAborted()); + }; + if (signal) signal.addEventListener("abort", onAbort); + + onDone = () => { + clearTimeout(timeoutId); + clearInterval(intervalId); + if (signal) signal.removeEventListener("abort", onAbort); + }; + }); +} diff --git a/packages/utils/test/unit/waitFor.test.ts b/packages/utils/test/unit/waitFor.test.ts new file mode 100644 index 000000000000..1dd3dec766b7 --- /dev/null +++ b/packages/utils/test/unit/waitFor.test.ts @@ -0,0 +1,37 @@ +import "../setup.js"; +import {expect} from "chai"; +import {waitFor} from "../../src/waitFor.js"; +import {ErrorAborted, TimeoutError} from "../../src/errors.js"; + +describe("waitFor", () => { + const interval = 10; + const timeout = 20; + + it("Should resolve if condition is already true", async () => { + await expect(waitFor(() => true, {interval, timeout})).to.be.fulfilled; + }); + + it("Should resolve if condition becomes true within timeout", async () => { + let condition = false; + setTimeout(() => { + condition = true; + }, interval); + await waitFor(() => condition, {interval, timeout}); + }); + + it("Should reject with TimeoutError if condition does not become true within timeout", async () => { + await expect(waitFor(() => false, {interval, timeout})).to.be.rejectedWith(TimeoutError); + }); + + it("Should reject with ErrorAborted if aborted before condition becomes true", async () => { + const controller = new AbortController(); + setTimeout(() => controller.abort(), interval); + await expect(waitFor(() => false, {interval, timeout, signal: controller.signal})).to.be.rejectedWith(ErrorAborted); + }); + + it("Should reject with ErrorAborted if signal is already aborted", async () => { + const controller = new AbortController(); + controller.abort(); + await expect(waitFor(() => true, {interval, timeout, signal: controller.signal})).to.be.rejectedWith(ErrorAborted); + }); +}); From 7b5fc63e678c4d9010b8b6993a17a8db84667013 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 24 Jul 2023 19:06:26 +0200 Subject: [PATCH 86/96] fix: eventstream invalid topic error (#5787) --- packages/api/src/beacon/server/events.ts | 11 +++++++++-- packages/api/src/beacon/server/index.ts | 5 ++++- packages/api/src/utils/server/errors.ts | 9 +++++++++ packages/api/src/utils/server/index.ts | 1 + packages/beacon-node/src/api/impl/errors.ts | 10 ++-------- packages/beacon-node/src/api/impl/events/index.ts | 7 ------- 6 files changed, 25 insertions(+), 18 deletions(-) create mode 100644 packages/api/src/utils/server/errors.ts diff --git a/packages/api/src/beacon/server/events.ts b/packages/api/src/beacon/server/events.ts index 3e45b104ecb5..f687f21f874b 100644 --- a/packages/api/src/beacon/server/events.ts +++ b/packages/api/src/beacon/server/events.ts @@ -1,7 +1,7 @@ import {ChainForkConfig} from "@lodestar/config"; import {ErrorAborted} from "@lodestar/utils"; -import {Api, ReqTypes, routesData, getEventSerdes} from "../routes/events.js"; -import {ServerRoutes} from "../../utils/server/index.js"; +import {Api, ReqTypes, routesData, getEventSerdes, eventTypes} from "../routes/events.js"; +import {ApiError, ServerRoutes} from "../../utils/server/index.js"; import {ServerApi} from "../../interfaces.js"; export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerRoutes { @@ -15,6 +15,13 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR id: "eventstream", handler: async (req, res) => { + const validTopics = new Set(Object.values(eventTypes)); + for (const topic of req.query.topics) { + if (!validTopics.has(topic)) { + throw new ApiError(400, `Invalid topic: ${topic}`); + } + } + const controller = new AbortController(); try { diff --git a/packages/api/src/beacon/server/index.ts b/packages/api/src/beacon/server/index.ts index 4f90ed24dee7..6c1cc9c16a4b 100644 --- a/packages/api/src/beacon/server/index.ts +++ b/packages/api/src/beacon/server/index.ts @@ -1,6 +1,6 @@ import {ChainForkConfig} from "@lodestar/config"; import {Api} from "../routes/index.js"; -import {ServerInstance, ServerRoute, RouteConfig, registerRoute} from "../../utils/server/index.js"; +import {ApiError, ServerInstance, ServerRoute, RouteConfig, registerRoute} from "../../utils/server/index.js"; import {ServerApi} from "../../interfaces.js"; import * as beacon from "./beacon.js"; @@ -13,6 +13,9 @@ import * as node from "./node.js"; import * as proof from "./proof.js"; import * as validator from "./validator.js"; +// Re-export for usage in beacon-node +export {ApiError}; + // Re-export for convenience export {RouteConfig}; diff --git a/packages/api/src/utils/server/errors.ts b/packages/api/src/utils/server/errors.ts new file mode 100644 index 000000000000..ea075678f4f6 --- /dev/null +++ b/packages/api/src/utils/server/errors.ts @@ -0,0 +1,9 @@ +import {HttpErrorCodes} from "../client/httpStatusCode.js"; + +export class ApiError extends Error { + statusCode: HttpErrorCodes; + constructor(statusCode: HttpErrorCodes, message?: string) { + super(message); + this.statusCode = statusCode; + } +} diff --git a/packages/api/src/utils/server/index.ts b/packages/api/src/utils/server/index.ts index 5a0227a01916..2e17e9ee2a6f 100644 --- a/packages/api/src/utils/server/index.ts +++ b/packages/api/src/utils/server/index.ts @@ -1,3 +1,4 @@ export * from "./genericJsonServer.js"; export * from "./registerRoute.js"; +export * from "./errors.js"; export * from "./types.js"; diff --git a/packages/beacon-node/src/api/impl/errors.ts b/packages/beacon-node/src/api/impl/errors.ts index 7169b7138295..cc877de90a7a 100644 --- a/packages/beacon-node/src/api/impl/errors.ts +++ b/packages/beacon-node/src/api/impl/errors.ts @@ -1,12 +1,6 @@ -import {HttpErrorCodes} from "@lodestar/api"; +import {ApiError} from "@lodestar/api/beacon/server"; -export class ApiError extends Error { - statusCode: HttpErrorCodes; - constructor(statusCode: HttpErrorCodes, message?: string) { - super(message); - this.statusCode = statusCode; - } -} +export {ApiError}; export class StateNotFound extends ApiError { constructor() { diff --git a/packages/beacon-node/src/api/impl/events/index.ts b/packages/beacon-node/src/api/impl/events/index.ts index 18f9ff160b3a..9f75bc0a83c5 100644 --- a/packages/beacon-node/src/api/impl/events/index.ts +++ b/packages/beacon-node/src/api/impl/events/index.ts @@ -1,19 +1,12 @@ import {routes, ServerApi} from "@lodestar/api"; import {ApiModules} from "../types.js"; -import {ApiError} from "../errors.js"; export function getEventsApi({chain}: Pick): ServerApi { - const validTopics = new Set(Object.values(routes.events.eventTypes)); - return { async eventstream(topics, signal, onEvent) { const onAbortFns: (() => void)[] = []; for (const topic of topics) { - if (!validTopics.has(topic)) { - throw new ApiError(400, `Unknown topic ${topic}`); - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any const handler = (data: any): void => { // TODO: What happens if this handler throws? Does it break the other chain.emitter listeners? From 056366142798c05a15beea20c16528af97d9ea61 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 24 Jul 2023 23:23:04 +0200 Subject: [PATCH 87/96] chore: remove redundant eventstream error listener (#5798) --- packages/api/src/beacon/server/events.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/api/src/beacon/server/events.ts b/packages/api/src/beacon/server/events.ts index f687f21f874b..834b6b67fca4 100644 --- a/packages/api/src/beacon/server/events.ts +++ b/packages/api/src/beacon/server/events.ts @@ -1,5 +1,4 @@ import {ChainForkConfig} from "@lodestar/config"; -import {ErrorAborted} from "@lodestar/utils"; import {Api, ReqTypes, routesData, getEventSerdes, eventTypes} from "../routes/events.js"; import {ApiError, ServerRoutes} from "../../utils/server/index.js"; import {ServerApi} from "../../interfaces.js"; @@ -57,12 +56,6 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR // The client may disconnect and we need to clean the subscriptions. req.socket.once("close", () => resolve()); req.socket.once("end", () => resolve()); - req.raw.once("error", (err) => { - if ((err as unknown as {code: string}).code === "ECONNRESET") { - return reject(new ErrorAborted()); - } - return reject(err); - }); }); // api.eventstream will never stop, so no need to ever call `res.raw.end();` From ab285d3836a1ac1808ef198e89f09ff88c2c27c5 Mon Sep 17 00:00:00 2001 From: Tyler <122291810+0xTylerHolmes@users.noreply.github.com> Date: Tue, 25 Jul 2023 15:40:00 +0200 Subject: [PATCH 88/96] feat: code coverage for tests using c8 (#5225) * add support for code coverage reports using c8 * chore: update lockfile --------- Co-authored-by: Cayman --- .c8rc.json | 21 ++++++++++++++++++ package.json | 6 ++++++ yarn.lock | 61 +++++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 .c8rc.json diff --git a/.c8rc.json b/.c8rc.json new file mode 100644 index 000000000000..51d1ef9902c4 --- /dev/null +++ b/.c8rc.json @@ -0,0 +1,21 @@ +{ + "all": true, + "cache": false, + "extension": [".ts", ".js", ".jsx", ".tsx"], + "include": [ + "**/packages/**" + ], + "exclude": [ + "**/node_modules/**", + "**/test/**" + ], + "instrument": true, + "sourceMap": true, + "check-coverage": true, + "reporter": [ + "html", + "lcov", + "text", + "text-summary" + ] +} diff --git a/package.json b/package.json index 4b7d0fc5268f..b467c150a35f 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,11 @@ "test:e2e": "lerna run test:e2e --no-bail --concurrency 1", "test:e2e:sim": "lerna run test:e2e:sim --no-bail", "test:spec": "lerna run test:spec --no-bail", + "test-coverage:unit": "c8 --config .c8rc.json --report-dir coverage/unit/ --all npm run test:unit", + "test-coverage:browsers": "c8 --config .c8rc.json --report-dir coverage/browsers/ --all npm run test:browsers", + "test-coverage:e2e": "c8 --config .c8rc.json --report-dir coverage/e2e/ --all npm run test:e2e", + "test-coverage:e2e-sim": "c8 --config .c8rc.json --report-dir coverage/e2e-sim/ --all npm run test:e2e:sim", + "test-coverage:spec": "c8 --config .c8rc.json --report-dir coverage/spec/ --all npm run test:spec", "benchmark": "yarn benchmark:files 'packages/*/test/perf/**/*.test.ts'", "benchmark:files": "LODESTAR_PRESET=mainnet NODE_OPTIONS='--max-old-space-size=4096 --loader=ts-node/esm' benchmark --config .benchrc.yaml --defaultBranch unstable", "release:create-rc": "node scripts/release/create_rc.mjs", @@ -46,6 +51,7 @@ "@types/sinon-chai": "^3.2.9", "@typescript-eslint/eslint-plugin": "6.0.0", "@typescript-eslint/parser": "6.0.0", + "c8": "^7.13.0", "chai": "^4.3.7", "chai-as-promised": "^7.1.1", "codecov": "^3.8.3", diff --git a/yarn.lock b/yarn.lock index f8d1f788b660..ad622b9859c4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -426,6 +426,11 @@ resolved "https://registry.yarnpkg.com/@balena/dockerignore/-/dockerignore-1.0.2.tgz#9ffe4726915251e8eb69f44ef3547e0da2c03e0d" integrity sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q== +"@bcoe/v8-coverage@^0.2.3": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" + integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== + "@chainsafe/as-chacha20poly1305@^0.1.0": version "0.1.0" resolved "https://registry.yarnpkg.com/@chainsafe/as-chacha20poly1305/-/as-chacha20poly1305-0.1.0.tgz#7da6f8796f9b42dac6e830a086d964f1f9189e09" @@ -1676,7 +1681,7 @@ js-yaml "^3.13.1" resolve-from "^5.0.0" -"@istanbuljs/schema@^0.1.2": +"@istanbuljs/schema@^0.1.2", "@istanbuljs/schema@^0.1.3": version "0.1.3" resolved "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz" integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA== @@ -1711,7 +1716,7 @@ "@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/sourcemap-codec" "^1.4.10" -"@jridgewell/trace-mapping@^0.3.17": +"@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.17": version "0.3.18" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz#25783b2086daf6ff1dcb53c9249ae480e4dd4cd6" integrity sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA== @@ -3549,6 +3554,11 @@ "@types/through" "*" rxjs "^7.2.0" +"@types/istanbul-lib-coverage@^2.0.1": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz#8467d4b3c087805d63580480890791277ce35c44" + integrity sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g== + "@types/js-yaml@^4.0.5": version "4.0.5" resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.5.tgz#738dd390a6ecc5442f35e7f03fa1431353f7e138" @@ -5247,6 +5257,24 @@ c-kzg@^2.1.0: bindings "^1.5.0" node-addon-api "^5.0.0" +c8@^7.13.0: + version "7.14.0" + resolved "https://registry.yarnpkg.com/c8/-/c8-7.14.0.tgz#f368184c73b125a80565e9ab2396ff0be4d732f3" + integrity sha512-i04rtkkcNcCf7zsQcSv/T9EbUn4RXQ6mropeMcjFOsQXQ0iGLAr/xT6TImQg4+U9hmNpN9XdvPkjUL1IzbgxJw== + dependencies: + "@bcoe/v8-coverage" "^0.2.3" + "@istanbuljs/schema" "^0.1.3" + find-up "^5.0.0" + foreground-child "^2.0.0" + istanbul-lib-coverage "^3.2.0" + istanbul-lib-report "^3.0.0" + istanbul-reports "^3.1.4" + rimraf "^3.0.2" + test-exclude "^6.0.0" + v8-to-istanbul "^9.0.0" + yargs "^16.2.0" + yargs-parser "^20.2.9" + cacache@^15.2.0: version "15.3.0" resolved "https://registry.yarnpkg.com/cacache/-/cacache-15.3.0.tgz#dc85380fb2f556fe3dda4c719bfa0ec875a7f1eb" @@ -5993,6 +6021,11 @@ conventional-recommended-bump@6.1.0: meow "^8.0.0" q "^1.5.1" +convert-source-map@^1.6.0: + version "1.9.0" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f" + integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A== + convert-source-map@^1.7.0: version "1.7.0" resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz" @@ -9532,6 +9565,11 @@ istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.0.0-alpha.1: resolved "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.0.0.tgz" integrity sha512-UiUIqxMgRDET6eR+o5HbfRYP1l0hqkWOs7vNxC/mggutCMUIhWMm8gAHb8tHlyfD3/l6rlgNA5cKdDzEAf6hEg== +istanbul-lib-coverage@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz#189e7909d0a39fa5a3dfad5b03f71947770191d3" + integrity sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw== + istanbul-lib-hook@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-3.0.0.tgz" @@ -9588,6 +9626,14 @@ istanbul-reports@^3.0.2: html-escaper "^2.0.0" istanbul-lib-report "^3.0.0" +istanbul-reports@^3.1.4: + version "3.1.5" + resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-3.1.5.tgz#cc9a6ab25cb25659810e4785ed9d9fb742578bae" + integrity sha512-nUsEMa9pBt/NOHqbcbeJEgqIlY/K7rVWUX6Lql2orY5e9roQOthbR3vtY4zzf2orPELg80fnxxk9zUyPlgwD1w== + dependencies: + html-escaper "^2.0.0" + istanbul-lib-report "^3.0.0" + it-all@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/it-all/-/it-all-2.0.0.tgz#6f4e5cdb71af02793072822a90bc44de901a92c3" @@ -15238,6 +15284,15 @@ v8-compile-cache@2.3.0: resolved "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz" integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== +v8-to-istanbul@^9.0.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-9.1.0.tgz#1b83ed4e397f58c85c266a570fc2558b5feb9265" + integrity sha512-6z3GW9x8G1gd+JIIgQQQxXuiJtCXeAjp6RaPEPLv62mH3iPHPxV6W3robxtCzNErRo6ZwTmzWhsbNvjyEBKzKA== + dependencies: + "@jridgewell/trace-mapping" "^0.3.12" + "@types/istanbul-lib-coverage" "^2.0.1" + convert-source-map "^1.6.0" + validate-npm-package-license@3.0.4, validate-npm-package-license@^3.0.1, validate-npm-package-license@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a" @@ -16003,7 +16058,7 @@ yargs-parser@^18.1.2: camelcase "^5.0.0" decamelize "^1.2.0" -yargs-parser@^20.2.2, yargs-parser@^20.2.3: +yargs-parser@^20.2.2, yargs-parser@^20.2.3, yargs-parser@^20.2.9: version "20.2.9" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== From 11bbca15266fed24842c0f63d2dc549f9cb3b007 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 25 Jul 2023 14:05:18 -0400 Subject: [PATCH 89/96] fix: prevent non-local IP enr warning (#5802) --- packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts b/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts index 3c7053190335..9313b1f47f88 100644 --- a/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts +++ b/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts @@ -54,13 +54,13 @@ export function isLocalMultiAddr(multiaddr: Multiaddr | undefined): boolean { } export function overwriteEnrWithCliArgs(enr: SignableENR, args: BeaconArgs, logger: Logger): void { - const {listenAddress, port, discoveryPort, listenAddress6, port6, discoveryPort6} = parseListenArgs(args); - enr.ip = args["enr.ip"] ?? listenAddress; - enr.tcp = args["enr.tcp"] ?? port; - enr.udp = args["enr.udp"] ?? discoveryPort; - enr.ip6 = args["enr.ip6"] ?? listenAddress6; - enr.tcp6 = args["enr.tcp6"] ?? port6; - enr.udp6 = args["enr.udp6"] ?? discoveryPort6; + const {port, discoveryPort, port6, discoveryPort6} = parseListenArgs(args); + enr.ip = args["enr.ip"] ?? enr.ip; + enr.tcp = args["enr.tcp"] ?? port ?? enr.tcp; + enr.udp = args["enr.udp"] ?? discoveryPort ?? enr.udp; + enr.ip6 = args["enr.ip6"] ?? enr.ip6; + enr.tcp6 = args["enr.tcp6"] ?? port6 ?? enr.tcp6; + enr.udp6 = args["enr.udp6"] ?? discoveryPort6 ?? enr.udp6; function testMultiaddrForLocal(mu: Multiaddr, ip4: boolean): void { const isLocal = isLocalMultiAddr(mu); From c112f016df2a441219df3a93d919276066a68a95 Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Wed, 26 Jul 2023 10:47:03 +0700 Subject: [PATCH 90/96] feat: only dial subnet peers if needed (#5782) * fix: do not dial more peers if node has enough subnet peers * fix: reorder conditions in shouldDialPeer --- .../beacon-node/src/network/core/metrics.ts | 10 +++ .../beacon-node/src/network/peers/discover.ts | 61 ++++++++++++++----- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index a7ff827fd712..92791ec00d5a 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -115,6 +115,16 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { name: "lodestar_discovery_peers_to_connect", help: "Current peers to connect count from discoverPeers requests", }), + subnetPeersToConnect: register.gauge<"type">({ + name: "lodestar_discovery_subnet_peers_to_connect", + help: "Current peers to connect count from discoverPeers requests", + labelNames: ["type"], + }), + subnetsToConnect: register.gauge<"type">({ + name: "lodestar_discovery_subnets_to_connect", + help: "Current subnets to connect count from discoverPeers requests", + labelNames: ["type"], + }), cachedENRsSize: register.gauge({ name: "lodestar_discovery_cached_enrs_size", help: "Current size of the cachedENRs Set", diff --git a/packages/beacon-node/src/network/peers/discover.ts b/packages/beacon-node/src/network/peers/discover.ts index fe9aefb5a949..84baf67ebbd4 100644 --- a/packages/beacon-node/src/network/peers/discover.ts +++ b/packages/beacon-node/src/network/peers/discover.ts @@ -55,6 +55,16 @@ enum DiscoveredPeerStatus { } type UnixMs = number; +/** + * Maintain peersToConnect to avoid having too many topic peers at some point. + * See https://github.com/ChainSafe/lodestar/issues/5741#issuecomment-1643113577 + */ +type SubnetRequestInfo = { + toUnixMs: UnixMs; + // when node is stable this should be 0 + peersToConnect: number; +}; + export type SubnetDiscvQueryMs = { subnet: number; type: SubnetType; @@ -83,9 +93,9 @@ export class PeerDiscovery { private cachedENRs = new Map(); private randomNodeQuery: QueryStatus = {code: QueryStatusCode.NotActive}; private peersToConnect = 0; - private subnetRequests: Record> = { + private subnetRequests: Record> = { attnets: new Map(), - syncnets: new Map([[10, Date.now() + 2 * 60 * 60 * 1000]]), + syncnets: new Map(), }; /** The maximum number of peers we allow (exceptions for subnet peers) */ @@ -134,6 +144,14 @@ export class PeerDiscovery { metrics.discovery.cachedENRsSize.addCollect(() => { metrics.discovery.cachedENRsSize.set(this.cachedENRs.size); metrics.discovery.peersToConnect.set(this.peersToConnect); + for (const type of [SubnetType.attnets, SubnetType.syncnets]) { + const subnetPeersToConnect = Array.from(this.subnetRequests[type].values()).reduce( + (acc, {peersToConnect}) => acc + peersToConnect, + 0 + ); + metrics.discovery.subnetPeersToConnect.set({type}, subnetPeersToConnect); + metrics.discovery.subnetsToConnect.set({type}, this.subnetRequests[type].size); + } }); } } @@ -186,12 +204,6 @@ export class PeerDiscovery { this.peersToConnect += peersToConnect; subnet: for (const subnetRequest of subnetRequests) { - // Extend the toUnixMs for this subnet - const prevUnixMs = this.subnetRequests[subnetRequest.type].get(subnetRequest.subnet); - if (prevUnixMs === undefined || prevUnixMs < subnetRequest.toUnixMs) { - this.subnetRequests[subnetRequest.type].set(subnetRequest.subnet, subnetRequest.toUnixMs); - } - // Get cached ENRs from the discovery service that are in the requested `subnetId`, but not connected yet let cachedENRsInSubnet = 0; for (const cachedENR of cachedENRsReverse) { @@ -204,6 +216,17 @@ export class PeerDiscovery { } } + const subnetPeersToConnect = Math.max(subnetRequest.maxPeersToDiscover - cachedENRsInSubnet, 0); + + // Extend the toUnixMs for this subnet + const prevUnixMs = this.subnetRequests[subnetRequest.type].get(subnetRequest.subnet)?.toUnixMs; + const newUnixMs = + prevUnixMs !== undefined && prevUnixMs > subnetRequest.toUnixMs ? prevUnixMs : subnetRequest.toUnixMs; + this.subnetRequests[subnetRequest.type].set(subnetRequest.subnet, { + toUnixMs: newUnixMs, + peersToConnect: subnetPeersToConnect, + }); + // Query a discv5 query if more peers are needed subnetsToDiscoverPeers.push(subnetRequest); } @@ -372,23 +395,31 @@ export class PeerDiscovery { } private shouldDialPeer(peer: CachedENR): boolean { - if (this.peersToConnect > 0) { - return true; - } - for (const type of [SubnetType.attnets, SubnetType.syncnets]) { - for (const [subnet, toUnixMs] of this.subnetRequests[type].entries()) { - if (toUnixMs < Date.now()) { - // Prune all requests + for (const [subnet, {toUnixMs, peersToConnect}] of this.subnetRequests[type].entries()) { + if (toUnixMs < Date.now() || peersToConnect === 0) { + // Prune all requests so that we don't have to loop again + // if we have low subnet peers then PeerManager will update us again with subnet + toUnixMs + peersToConnect this.subnetRequests[type].delete(subnet); } else { + // not expired and peersToConnect > 0 + // if we have enough subnet peers, no need to dial more or we may have performance issues + // see https://github.com/ChainSafe/lodestar/issues/5741#issuecomment-1643113577 if (peer.subnets[type][subnet]) { + this.subnetRequests[type].set(subnet, {toUnixMs, peersToConnect: Math.max(peersToConnect - 1, 0)}); return true; } } } } + // ideally we may want to leave this cheap condition at the top of the function + // however we want to also update peersToConnect in this.subnetRequests + // the this.subnetRequests[type] gradually has 0 subnet so this function should be cheap enough + if (this.peersToConnect > 0) { + return true; + } + return false; } From 26fd079206f6e776c114f111de05019916a8e132 Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Wed, 26 Jul 2023 10:49:33 +0700 Subject: [PATCH 91/96] feat: poll proposer duties of next epoch in advance (#5794) * feat: poll proposer duties of next epoch in advance * chore: refactor function name msToNextEpoch * fix: pollBeaconProposersNextEpoch * fix: set MAX_API_CLOCK_DISPARITY_SEC to 0.5s * chore: add more comments * feat: wait for checkpoint state up to 1 slot time --- .../src/api/impl/validator/index.ts | 78 ++++++++++++++++--- .../beacon-node/src/chain/prepareNextSlot.ts | 2 +- .../src/metrics/metrics/lodestar.ts | 11 +++ packages/validator/src/services/block.ts | 1 + .../validator/src/services/blockDuties.ts | 40 ++++++++-- .../test/unit/services/blockDuties.test.ts | 31 +++++++- 6 files changed, 143 insertions(+), 20 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index bc0b25a953d6..d8d405f334f0 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -32,6 +32,8 @@ import {ApiModules} from "../types.js"; import {RegenCaller} from "../../../chain/regen/index.js"; import {getValidatorStatus} from "../beacon/state/utils.js"; import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js"; +import {SCHEDULER_LOOKAHEAD_FACTOR} from "../../../chain/prepareNextSlot.js"; +import {ChainEvent, CheckpointHex} from "../../../chain/index.js"; import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices} from "./utils.js"; /** @@ -66,9 +68,10 @@ export function getValidatorApi({ /** * Validator clock may be advanced from beacon's clock. If the validator requests a resource in a * future slot, wait some time instead of rejecting the request because it's in the future. + * This value is the same to MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC. * For very fast networks, reduce clock disparity to half a slot. */ - const MAX_API_CLOCK_DISPARITY_SEC = Math.min(1, config.SECONDS_PER_SLOT / 2); + const MAX_API_CLOCK_DISPARITY_SEC = Math.min(0.5, config.SECONDS_PER_SLOT / 2); const MAX_API_CLOCK_DISPARITY_MS = MAX_API_CLOCK_DISPARITY_SEC * 1000; /** Compute and cache the genesis block root */ @@ -118,19 +121,55 @@ export function getValidatorApi({ * Prevents a validator from not being able to get the attestater duties correctly if the beacon and validator clocks are off */ async function waitForNextClosestEpoch(): Promise { + const toNextEpochMs = msToNextEpoch(); + if (toNextEpochMs > 0 && toNextEpochMs < MAX_API_CLOCK_DISPARITY_MS) { + const nextEpoch = chain.clock.currentEpoch + 1; + await chain.clock.waitForSlot(computeStartSlotAtEpoch(nextEpoch)); + } + } + + /** + * Compute ms to the next epoch. + */ + function msToNextEpoch(): number { const nextEpoch = chain.clock.currentEpoch + 1; const secPerEpoch = SLOTS_PER_EPOCH * config.SECONDS_PER_SLOT; const nextEpochStartSec = chain.genesisTime + nextEpoch * secPerEpoch; - const msToNextEpoch = nextEpochStartSec * 1000 - Date.now(); - if (msToNextEpoch > 0 && msToNextEpoch < MAX_API_CLOCK_DISPARITY_MS) { - await chain.clock.waitForSlot(computeStartSlotAtEpoch(nextEpoch)); - } + return nextEpochStartSec * 1000 - Date.now(); } function currentEpochWithDisparity(): Epoch { return computeEpochAtSlot(getCurrentSlot(config, chain.genesisTime - MAX_API_CLOCK_DISPARITY_SEC)); } + /** + * This function is called 1s before next epoch, usually at that time PrepareNextSlotScheduler finishes + * so we should have checkpoint state, otherwise wait for up to `timeoutMs`. + */ + async function waitForCheckpointState( + cpHex: CheckpointHex, + timeoutMs: number + ): Promise { + const cpState = chain.regen.getCheckpointStateSync(cpHex); + if (cpState) { + return cpState; + } + const cp = { + epoch: cpHex.epoch, + root: fromHexString(cpHex.rootHex), + }; + // if not, wait for ChainEvent.checkpoint event until timeoutMs + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(null), timeoutMs); + chain.emitter.on(ChainEvent.checkpoint, (eventCp, cpState) => { + if (ssz.phase0.Checkpoint.equals(eventCp, cp)) { + clearTimeout(timer); + resolve(cpState); + } + }); + }); + } + /** * Reject any request while the node is syncing */ @@ -387,15 +426,32 @@ export function getValidatorApi({ // Early check that epoch is within [current_epoch, current_epoch + 1], or allow for pre-genesis const currentEpoch = currentEpochWithDisparity(); - if (currentEpoch >= 0 && epoch !== currentEpoch && epoch !== currentEpoch + 1) { - throw Error(`Requested epoch ${epoch} must equal current ${currentEpoch} or next epoch ${currentEpoch + 1}`); + const nextEpoch = currentEpoch + 1; + if (currentEpoch >= 0 && epoch !== currentEpoch && epoch !== nextEpoch) { + throw Error(`Requested epoch ${epoch} must equal current ${currentEpoch} or next epoch ${nextEpoch}`); } - // May request for an epoch that's in the future, for getBeaconProposersNextEpoch() - await waitForNextClosestEpoch(); - const head = chain.forkChoice.getHead(); - const state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties); + let state: CachedBeaconStateAllForks | undefined = undefined; + const slotMs = config.SECONDS_PER_SLOT * 1000; + const prepareNextSlotLookAheadMs = slotMs / SCHEDULER_LOOKAHEAD_FACTOR; + const toNextEpochMs = msToNextEpoch(); + // validators may request next epoch's duties when it's close to next epoch + // this is to avoid missed block proposal due to 0 epoch look ahead + if (epoch === nextEpoch && toNextEpochMs < prepareNextSlotLookAheadMs) { + // wait for maximum 1 slot for cp state which is the timeout of validator api + const cpState = await waitForCheckpointState({rootHex: head.blockRoot, epoch}, slotMs); + if (cpState) { + state = cpState; + metrics?.duties.requestNextEpochProposalDutiesHit.inc(); + } else { + metrics?.duties.requestNextEpochProposalDutiesMiss.inc(); + } + } + + if (!state) { + state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties); + } const stateEpoch = state.epochCtx.epoch; let indexes: ValidatorIndex[] = []; diff --git a/packages/beacon-node/src/chain/prepareNextSlot.ts b/packages/beacon-node/src/chain/prepareNextSlot.ts index 38a97efd38a2..1091fd716b60 100644 --- a/packages/beacon-node/src/chain/prepareNextSlot.ts +++ b/packages/beacon-node/src/chain/prepareNextSlot.ts @@ -14,7 +14,7 @@ import {IBeaconChain} from "./interface.js"; import {RegenCaller} from "./regen/index.js"; /* With 12s slot times, this scheduler will run 4s before the start of each slot (`12 / 3 = 4`). */ -const SCHEDULER_LOOKAHEAD_FACTOR = 3; +export const SCHEDULER_LOOKAHEAD_FACTOR = 3; /* We don't want to do more epoch transition than this */ const PREPARE_EPOCH_LIMIT = 1; diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 04915a064899..8b3410a6eadf 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -245,6 +245,17 @@ export function createLodestarMetrics( }), }, + duties: { + requestNextEpochProposalDutiesHit: register.gauge({ + name: "lodestar_duties_request_next_epoch_proposal_duties_hit_total", + help: "Total count of requestNextEpochProposalDuties hit", + }), + requestNextEpochProposalDutiesMiss: register.gauge({ + name: "lodestar_duties_request_next_epoch_proposal_duties_miss_total", + help: "Total count of requestNextEpochProposalDuties miss", + }), + }, + // Beacon state transition metrics epochTransitionTime: register.histogram({ diff --git a/packages/validator/src/services/block.ts b/packages/validator/src/services/block.ts index 2627bb1892e1..c1385fce9a9c 100644 --- a/packages/validator/src/services/block.ts +++ b/packages/validator/src/services/block.ts @@ -69,6 +69,7 @@ export class BlockProposingService { private readonly metrics: Metrics | null ) { this.dutiesService = new BlockDutiesService( + config, logger, api, clock, diff --git a/packages/validator/src/services/blockDuties.ts b/packages/validator/src/services/blockDuties.ts index d0812f173c9e..65b98ce3b1f0 100644 --- a/packages/validator/src/services/blockDuties.ts +++ b/packages/validator/src/services/blockDuties.ts @@ -1,12 +1,19 @@ import {toHexString} from "@chainsafe/ssz"; -import {computeEpochAtSlot} from "@lodestar/state-transition"; +import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {BLSPubkey, Epoch, RootHex, Slot} from "@lodestar/types"; import {Api, ApiError, routes} from "@lodestar/api"; +import {sleep} from "@lodestar/utils"; +import {ChainConfig} from "@lodestar/config"; import {IClock, differenceHex, LoggerVc} from "../util/index.js"; import {PubkeyHex} from "../types.js"; import {Metrics} from "../metrics.js"; import {ValidatorStore} from "./validatorStore.js"; +/** This polls block duties 1s before the next epoch */ +// TODO: change to 6 to do it 2s before the next epoch +// once we have some improvement on epoch transition time +// see https://github.com/ChainSafe/lodestar/issues/5792#issuecomment-1647457442 +const BLOCK_DUTIES_LOOKAHEAD_FACTOR = 12; /** Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch */ const HISTORICAL_DUTIES_EPOCHS = 2; // Re-declaring to not have to depend on `lodestar-params` just for this 0 @@ -24,9 +31,10 @@ export class BlockDutiesService { private readonly proposers = new Map(); constructor( + private readonly config: ChainConfig, private readonly logger: LoggerVc, private readonly api: Api, - clock: IClock, + private readonly clock: IClock, private readonly validatorStore: ValidatorStore, private readonly metrics: Metrics | null, notifyBlockProductionFn: NotifyBlockProductionFn @@ -75,7 +83,7 @@ export class BlockDutiesService { } } - private runBlockDutiesTask = async (slot: Slot): Promise => { + private runBlockDutiesTask = async (slot: Slot, signal: AbortSignal): Promise => { try { if (slot < 0) { // Before genesis, fetch the genesis duties but don't notify block production @@ -84,7 +92,7 @@ export class BlockDutiesService { await this.pollBeaconProposers(GENESIS_EPOCH); } } else { - await this.pollBeaconProposersAndNotify(slot); + await this.pollBeaconProposersAndNotify(slot, signal); } } catch (e) { this.logger.error("Error on pollBeaconProposers", {}, e as Error); @@ -117,8 +125,17 @@ export class BlockDutiesService { * through the slow path every time. I.e., the proposal will only happen after we've been able to * download and process the duties from the BN. This means it is very important to ensure this * function is as fast as possible. + * - Starting from Jul 2023, we poll proposers 1s before the next epoch thanks to PrepareNextSlotScheduler + * usually finishes in 3s. */ - private async pollBeaconProposersAndNotify(currentSlot: Slot): Promise { + private async pollBeaconProposersAndNotify(currentSlot: Slot, signal: AbortSignal): Promise { + const nextEpoch = computeEpochAtSlot(currentSlot) + 1; + const isLastSlotEpoch = computeStartSlotAtEpoch(nextEpoch) === currentSlot + 1; + if (isLastSlotEpoch) { + // no need to await for other steps, just poll proposers for next epoch + void this.pollBeaconProposersNextEpoch(currentSlot, nextEpoch, signal); + } + // Notify the block proposal service for any proposals that we have in our cache. const initialBlockProposers = this.getblockProposersAtSlot(currentSlot); if (initialBlockProposers.length > 0) { @@ -145,6 +162,19 @@ export class BlockDutiesService { } } + /** + * This is to avoid some delay on the first slot of the opoch when validators has proposal duties. + * See https://github.com/ChainSafe/lodestar/issues/5792 + */ + private async pollBeaconProposersNextEpoch(currentSlot: Slot, nextEpoch: Epoch, signal: AbortSignal): Promise { + const nextSlot = currentSlot + 1; + const lookAheadMs = (this.config.SECONDS_PER_SLOT * 1000) / BLOCK_DUTIES_LOOKAHEAD_FACTOR; + await sleep(this.clock.msToSlot(nextSlot) - lookAheadMs, signal); + this.logger.debug("Polling proposers for next epoch", {nextEpoch, nextSlot}); + // Poll proposers for the next epoch + await this.pollBeaconProposers(nextEpoch); + } + private async pollBeaconProposers(epoch: Epoch): Promise { // Only download duties and push out additional block production events if we have some validators. if (!this.validatorStore.hasSomeValidators()) { diff --git a/packages/validator/test/unit/services/blockDuties.test.ts b/packages/validator/test/unit/services/blockDuties.test.ts index fa22984bc59a..d6152f6d7b09 100644 --- a/packages/validator/test/unit/services/blockDuties.test.ts +++ b/packages/validator/test/unit/services/blockDuties.test.ts @@ -5,6 +5,7 @@ import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; import {RootHex} from "@lodestar/types"; import {HttpStatusCode, routes} from "@lodestar/api"; +import {chainConfig} from "@lodestar/config/default"; import {toHex} from "@lodestar/utils"; import {BlockDutiesService} from "../../../src/services/blockDuties.js"; import {ValidatorStore} from "../../../src/services/validatorStore.js"; @@ -49,7 +50,15 @@ describe("BlockDutiesService", function () { const notifyBlockProductionFn = sinon.stub(); // Returns void const clock = new ClockMock(); - const dutiesService = new BlockDutiesService(loggerVc, api, clock, validatorStore, null, notifyBlockProductionFn); + const dutiesService = new BlockDutiesService( + chainConfig, + loggerVc, + api, + clock, + validatorStore, + null, + notifyBlockProductionFn + ); // Trigger clock onSlot for slot 0 await clock.tickSlotFns(0, controller.signal); @@ -84,7 +93,15 @@ describe("BlockDutiesService", function () { // Clock will call runAttesterDutiesTasks() immediately const clock = new ClockMock(); - const dutiesService = new BlockDutiesService(loggerVc, api, clock, validatorStore, null, notifyBlockProductionFn); + const dutiesService = new BlockDutiesService( + chainConfig, + loggerVc, + api, + clock, + validatorStore, + null, + notifyBlockProductionFn + ); // Trigger clock onSlot for slot 0 api.validator.getProposerDuties.resolves({ @@ -151,7 +168,15 @@ describe("BlockDutiesService", function () { const notifyBlockProductionFn = sinon.stub(); // Returns void const clock = new ClockMock(); - const dutiesService = new BlockDutiesService(loggerVc, api, clock, validatorStore, null, notifyBlockProductionFn); + const dutiesService = new BlockDutiesService( + chainConfig, + loggerVc, + api, + clock, + validatorStore, + null, + notifyBlockProductionFn + ); // Trigger clock onSlot for slot 0 await clock.tickSlotFns(0, controller.signal); From 201dfc805e7f7a60c9b03474d6c96ba452ca7094 Mon Sep 17 00:00:00 2001 From: Cayman Date: Wed, 26 Jul 2023 13:40:36 -0400 Subject: [PATCH 92/96] chore: downgrade cross-fetch to v3 (#5806) --- packages/api/package.json | 2 +- packages/beacon-node/package.json | 2 +- packages/light-client/package.json | 2 +- packages/validator/package.json | 2 +- yarn.lock | 15 ++++----------- 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index e8146ea49d06..3bd8e9d3c47e 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -75,7 +75,7 @@ "@lodestar/params": "^1.9.2", "@lodestar/types": "^1.9.2", "@lodestar/utils": "^1.9.2", - "cross-fetch": "^4.0.0", + "cross-fetch": "^3.1.8", "eventsource": "^2.0.2", "qs": "^6.11.1" }, diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index e2f59e0f0fb9..96f37ce38366 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -136,7 +136,7 @@ "@types/datastore-level": "^3.0.0", "buffer-xor": "^2.0.2", "c-kzg": "^2.1.0", - "cross-fetch": "^4.0.0", + "cross-fetch": "^3.1.8", "datastore-core": "^9.1.1", "datastore-level": "^10.1.1", "deepmerge": "^4.3.1", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index cdb288dc33b5..6be35f6d578f 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -73,7 +73,7 @@ "@lodestar/state-transition": "^1.9.2", "@lodestar/types": "^1.9.2", "@lodestar/utils": "^1.9.2", - "cross-fetch": "^4.0.0", + "cross-fetch": "^3.1.8", "mitt": "^3.0.0", "strict-event-emitter-types": "^2.0.0" }, diff --git a/packages/validator/package.json b/packages/validator/package.json index 68fcead3dfe3..c199ceb2dbca 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -58,7 +58,7 @@ "@lodestar/types": "^1.9.2", "@lodestar/utils": "^1.9.2", "bigint-buffer": "^1.1.5", - "cross-fetch": "^4.0.0", + "cross-fetch": "^3.1.8", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index ad622b9859c4..6bc904fe509d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6154,17 +6154,10 @@ create-require@^1.1.0: resolved "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== -cross-fetch@^3.1.4: - version "3.1.5" - resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" - integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw== - dependencies: - node-fetch "2.6.7" - -cross-fetch@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-4.0.0.tgz#f037aef1580bb3a1a35164ea2a848ba81b445983" - integrity sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g== +cross-fetch@^3.1.4, cross-fetch@^3.1.8: + version "3.1.8" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82" + integrity sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg== dependencies: node-fetch "^2.6.12" From a752d158ca6bc95fa08f3d6dbf858b8d5fd88601 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 27 Jul 2023 12:51:27 -0400 Subject: [PATCH 93/96] chore: downgrade docker nodejs to 18 (#5812) --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5a9541e06f1a..346fffd117dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # --platform=$BUILDPLATFORM is used build javascript source with host arch # Otherwise TS builds on emulated archs and can be extremely slow (+1h) -FROM --platform=${BUILDPLATFORM:-amd64} node:20-alpine as build_src +FROM --platform=${BUILDPLATFORM:-amd64} node:18-alpine as build_src ARG COMMIT WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 && rm -rf /var/cache/apk/* @@ -21,7 +21,7 @@ RUN cd packages/cli && GIT_COMMIT=${COMMIT} yarn write-git-data # Copy built src + node_modules to build native packages for archs different than host. # Note: This step is redundant for the host arch -FROM node:20-alpine as build_deps +FROM node:18-alpine as build_deps WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 && rm -rf /var/cache/apk/* @@ -35,7 +35,7 @@ RUN cd node_modules/classic-level && yarn rebuild # Copy built src + node_modules to a new layer to prune unnecessary fs # Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020) -FROM node:20-alpine +FROM node:18-alpine WORKDIR /usr/app COPY --from=build_deps /usr/app . From 2d0557f4ee041d09f9aa22f63d40eba777eb95f1 Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Fri, 28 Jul 2023 01:29:07 +0700 Subject: [PATCH 94/96] feat: only subscribe to short lived subnets 2 slots in advanced (#5810) * feat: only subscribe to short lived subnets 2 slots in advanced * fix: dedup attestationPool.getAggregate() call * chore: fix comment in test --- .../src/api/impl/validator/index.ts | 2 +- .../src/network/subnets/attnetsService.ts | 38 ++++++++++--------- .../src/network/subnets/dllAttnetsService.ts | 37 +++++++++--------- .../network/subnets/dllAttnetsService.test.ts | 4 ++ 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index d8d405f334f0..00de92581476 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -603,7 +603,7 @@ export function getValidatorApi({ metrics?.production.producedAggregateParticipants.observe(aggregate.aggregationBits.getTrueBitIndexes().length); return { - data: chain.attestationPool.getAggregate(slot, attestationDataRoot), + data: aggregate, }; }, diff --git a/packages/beacon-node/src/network/subnets/attnetsService.ts b/packages/beacon-node/src/network/subnets/attnetsService.ts index 35f30eb41328..f49fd2ed0e88 100644 --- a/packages/beacon-node/src/network/subnets/attnetsService.ts +++ b/packages/beacon-node/src/network/subnets/attnetsService.ts @@ -39,6 +39,9 @@ enum SubnetSource { random = "random", } +/** As monitored on goerli, we only need to subscribe 2 slots before aggregator dutied slot to get stable mesh peers */ +const SLOTS_TO_SUBSCRIBE_IN_ADVANCE = 2; + /** * Manage random (long lived) subnets and committee (short lived) subnets. * - PeerManager uses attnetsService to know which peers are requried for duties @@ -118,7 +121,6 @@ export class AttnetsService implements IAttnetsService { addCommitteeSubscriptions(subscriptions: CommitteeSubscription[]): void { const currentSlot = this.clock.currentSlot; let addedknownValidators = false; - const subnetsToSubscribe: RequestedSubnet[] = []; for (const {validatorIndex, subnet, slot, isAggregator} of subscriptions) { // Add known validator @@ -129,23 +131,10 @@ export class AttnetsService implements IAttnetsService { this.committeeSubnets.request({subnet, toSlot: slot + 1}); if (isAggregator) { // need exact slot here - subnetsToSubscribe.push({subnet, toSlot: slot}); this.aggregatorSlotSubnet.getOrDefault(slot).add(subnet); } } - // Trigger gossip subscription first, in batch - if (subnetsToSubscribe.length > 0) { - this.subscribeToSubnets( - subnetsToSubscribe.map((sub) => sub.subnet), - SubnetSource.committee - ); - } - // Then, register the subscriptions - for (const subscription of subnetsToSubscribe) { - this.subscriptionsCommittee.request(subscription); - } - if (addedknownValidators) this.rebalanceRandomSubnets(); } @@ -179,16 +168,29 @@ export class AttnetsService implements IAttnetsService { /** * Run per slot. + * - Subscribe to gossip subnets `${SLOTS_TO_SUBSCRIBE_IN_ADVANCE}` slots in advance + * - Unsubscribe from expired subnets */ - private onSlot = (slot: Slot): void => { + private onSlot = (clockSlot: Slot): void => { try { + for (const [dutiedSlot, subnets] of this.aggregatorSlotSubnet.entries()) { + if (dutiedSlot === clockSlot + SLOTS_TO_SUBSCRIBE_IN_ADVANCE) { + // Trigger gossip subscription first, in batch + if (subnets.size > 0) { + this.subscribeToSubnets(Array.from(subnets), SubnetSource.committee); + } + // Then, register the subscriptions + Array.from(subnets).map((subnet) => this.subscriptionsCommittee.request({subnet, toSlot: dutiedSlot})); + } + } + // For node >= 64 validators, we should consistently subscribe to all subnets // it's important to check random subnets first // See https://github.com/ChainSafe/lodestar/issues/4929 - this.unsubscribeExpiredRandomSubnets(slot); - this.unsubscribeExpiredCommitteeSubnets(slot); + this.unsubscribeExpiredRandomSubnets(clockSlot); + this.unsubscribeExpiredCommitteeSubnets(clockSlot); } catch (e) { - this.logger.error("Error on AttnetsService.onSlot", {slot}, e as Error); + this.logger.error("Error on AttnetsService.onSlot", {slot: clockSlot}, e as Error); } }; diff --git a/packages/beacon-node/src/network/subnets/dllAttnetsService.ts b/packages/beacon-node/src/network/subnets/dllAttnetsService.ts index ae3abb8bbf91..08dd9e91da15 100644 --- a/packages/beacon-node/src/network/subnets/dllAttnetsService.ts +++ b/packages/beacon-node/src/network/subnets/dllAttnetsService.ts @@ -24,6 +24,9 @@ enum SubnetSource { longLived = "long_lived", } +/** As monitored on goerli, we only need to subscribe 2 slots before aggregator dutied slot to get stable mesh peers */ +const SLOTS_TO_SUBSCRIBE_IN_ADVANCE = 2; + /** * Manage deleterministic long lived (DLL) subnets and short lived subnets. * - PeerManager uses attnetsService to know which peers are required for duties and long lived subscriptions @@ -103,29 +106,14 @@ export class DLLAttnetsService implements IAttnetsService { * Called from the API when validator is a part of a committee. */ addCommitteeSubscriptions(subscriptions: CommitteeSubscription[]): void { - const subnetsToSubscribe: RequestedSubnet[] = []; - for (const {subnet, slot, isAggregator} of subscriptions) { // the peer-manager heartbeat will help find the subnet this.committeeSubnets.request({subnet, toSlot: slot + 1}); if (isAggregator) { // need exact slot here - subnetsToSubscribe.push({subnet, toSlot: slot}); this.aggregatorSlotSubnet.getOrDefault(slot).add(subnet); } } - - // Trigger gossip subscription first, in batch - if (subnetsToSubscribe.length > 0) { - this.subscribeToSubnets( - subnetsToSubscribe.map((sub) => sub.subnet), - SubnetSource.committee - ); - } - // Then, register the subscriptions - for (const subscription of subnetsToSubscribe) { - this.shortLivedSubscriptions.request(subscription); - } } /** @@ -167,12 +155,25 @@ export class DLLAttnetsService implements IAttnetsService { /** * Run per slot. + * - Subscribe to gossip subnets `${SLOTS_TO_SUBSCRIBE_IN_ADVANCE}` slots in advance + * - Unsubscribe from expired subnets */ - private onSlot = (slot: Slot): void => { + private onSlot = (clockSlot: Slot): void => { try { - this.unsubscribeExpiredCommitteeSubnets(slot); + for (const [dutiedSlot, subnets] of this.aggregatorSlotSubnet.entries()) { + if (dutiedSlot === clockSlot + SLOTS_TO_SUBSCRIBE_IN_ADVANCE) { + // Trigger gossip subscription first, in batch + if (subnets.size > 0) { + this.subscribeToSubnets(Array.from(subnets), SubnetSource.committee); + } + // Then, register the subscriptions + Array.from(subnets).map((subnet) => this.shortLivedSubscriptions.request({subnet, toSlot: dutiedSlot})); + } + } + + this.unsubscribeExpiredCommitteeSubnets(clockSlot); } catch (e) { - this.logger.error("Error on AttnetsService.onSlot", {slot}, e as Error); + this.logger.error("Error on AttnetsService.onSlot", {slot: clockSlot}, e as Error); } }; diff --git a/packages/beacon-node/test/unit/network/subnets/dllAttnetsService.test.ts b/packages/beacon-node/test/unit/network/subnets/dllAttnetsService.test.ts index 891c87918a91..9b402396843d 100644 --- a/packages/beacon-node/test/unit/network/subnets/dllAttnetsService.test.ts +++ b/packages/beacon-node/test/unit/network/subnets/dllAttnetsService.test.ts @@ -124,6 +124,10 @@ describe("DLLAttnetsService", () => { isAggregator: true, }; service.addCommitteeSubscriptions([subscription]); + // it does not subscribe immediately + expect(gossipStub.subscribeTopic.callCount).to.be.equal(SUBNETS_PER_NODE); + sandbox.clock.tick(config.SECONDS_PER_SLOT * (subscription.slot - 2) * 1000); + // then subscribe 2 slots before dutied slot expect(gossipStub.subscribeTopic.callCount).to.be.equal(SUBNETS_PER_NODE + 1); // then unsubscribe after the expiration sandbox.clock.tick(config.SECONDS_PER_SLOT * (subscription.slot + 1) * 1000); From 0c440c1a0d99bac880dabeb017f8e6a7e8f61404 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 27 Jul 2023 20:29:50 +0200 Subject: [PATCH 95/96] chore: add capella state transition perf tests (#5807) Add capella state transition perf tests --- .../test/perf/epoch/epochCapella.test.ts | 159 ++++++++++++++++++ packages/state-transition/test/perf/params.ts | 5 + .../test/utils/testFileCache.ts | 5 +- 3 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 packages/state-transition/test/perf/epoch/epochCapella.test.ts diff --git a/packages/state-transition/test/perf/epoch/epochCapella.test.ts b/packages/state-transition/test/perf/epoch/epochCapella.test.ts new file mode 100644 index 000000000000..86620fa2dfcf --- /dev/null +++ b/packages/state-transition/test/perf/epoch/epochCapella.test.ts @@ -0,0 +1,159 @@ +import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {ForkSeq} from "@lodestar/params"; +import { + computeStartSlotAtEpoch, + CachedBeaconStateAllForks, + CachedBeaconStateCapella, + CachedBeaconStateAltair, + beforeProcessEpoch, +} from "../../../src/index.js"; +import {getNetworkCachedState, beforeValue, LazyValue} from "../../utils/index.js"; +import {StateEpoch} from "../types.js"; +import {capellaState} from "../params.js"; +import {processJustificationAndFinalization} from "../../../src/epoch/processJustificationAndFinalization.js"; +import {processInactivityUpdates} from "../../../src/epoch/processInactivityUpdates.js"; +import {processRewardsAndPenalties} from "../../../src/epoch/processRewardsAndPenalties.js"; +import {processRegistryUpdates} from "../../../src/epoch/processRegistryUpdates.js"; +import {processSlashings} from "../../../src/epoch/processSlashings.js"; +import {processEth1DataReset} from "../../../src/epoch/processEth1DataReset.js"; +import {processEffectiveBalanceUpdates} from "../../../src/epoch/processEffectiveBalanceUpdates.js"; +import {processSlashingsReset} from "../../../src/epoch/processSlashingsReset.js"; +import {processRandaoMixesReset} from "../../../src/epoch/processRandaoMixesReset.js"; +import {processHistoricalRootsUpdate} from "../../../src/epoch/processHistoricalRootsUpdate.js"; +import {processParticipationFlagUpdates} from "../../../src/epoch/processParticipationFlagUpdates.js"; +import {processEpoch} from "../../../src/epoch/index.js"; + +const slot = computeStartSlotAtEpoch(capellaState.epoch) - 1; +const stateId = `${capellaState.network}_e${capellaState.epoch}`; +const fork = ForkSeq.altair; + +describe(`capella processEpoch - ${stateId}`, () => { + setBenchOpts({ + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 + }); + + const stateOg = beforeValue(async () => { + const state = await getNetworkCachedState(capellaState.network, slot, 300_000); + state.hashTreeRoot(); + return state; + }, 300_000); + + itBench({ + id: `capella processEpoch - ${stateId}`, + yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 + beforeEach: () => stateOg.value.clone(), + fn: (state) => { + const cache = beforeProcessEpoch(state); + processEpoch(fork, state as CachedBeaconStateCapella, cache); + state.epochCtx.afterProcessEpoch(state, cache); + // Simulate root computation through the next block to account for changes + // 74184 hash64 ops - 92.730 ms + state.hashTreeRoot(); + }, + }); + + // Only in local environment compute a full breakdown of the cost of each step + describe(`capella processEpoch steps - ${stateId}`, () => { + setBenchOpts({noThreshold: true}); + + benchmarkAltairEpochSteps(stateOg, stateId); + }); +}); + +function benchmarkAltairEpochSteps(stateOg: LazyValue, stateId: string): void { + const cache = beforeValue(() => beforeProcessEpoch(stateOg.value)); + + // const getPerfState = (): CachedBeaconStateCapella => { + // const state = originalState.clone(); + // state.setStateCachesAsTransient(); + // return state; + // }; + + itBench({ + id: `${stateId} - capella beforeProcessEpoch`, + fn: () => { + beforeProcessEpoch(stateOg.value); + }, + }); + + itBench({ + id: `${stateId} - capella processJustificationAndFinalization`, + beforeEach: () => stateOg.value.clone(), + fn: (state) => processJustificationAndFinalization(state, cache.value), + }); + + itBench({ + id: `${stateId} - capella processInactivityUpdates`, + beforeEach: () => stateOg.value.clone() as CachedBeaconStateAltair, + fn: (state) => processInactivityUpdates(state, cache.value), + }); + + itBench({ + id: `${stateId} - capella processRewardsAndPenalties`, + beforeEach: () => stateOg.value.clone() as CachedBeaconStateCapella, + fn: (state) => processRewardsAndPenalties(state, cache.value), + }); + + // TODO: Needs a better state to test with, current does not include enough actions: 17.715 us/op + itBench({ + id: `${stateId} - capella processRegistryUpdates`, + beforeEach: () => stateOg.value.clone(), + fn: (state) => processRegistryUpdates(state, cache.value), + }); + + // TODO: Needs a better state to test with, current does not include enough actions: 39.985 us/op + itBench({ + id: `${stateId} - capella processSlashings`, + beforeEach: () => stateOg.value.clone() as CachedBeaconStateCapella, + fn: (state) => processSlashings(state, cache.value), + }); + + itBench({ + id: `${stateId} - capella processEth1DataReset`, + beforeEach: () => stateOg.value.clone(), + fn: (state) => processEth1DataReset(state, cache.value), + }); + + itBench({ + id: `${stateId} - capella processEffectiveBalanceUpdates`, + beforeEach: () => stateOg.value.clone(), + fn: (state) => processEffectiveBalanceUpdates(state, cache.value), + }); + + itBench({ + id: `${stateId} - capella processSlashingsReset`, + beforeEach: () => stateOg.value.clone(), + fn: (state) => processSlashingsReset(state, cache.value), + }); + + itBench({ + id: `${stateId} - capella processRandaoMixesReset`, + beforeEach: () => stateOg.value.clone(), + fn: (state) => processRandaoMixesReset(state, cache.value), + }); + + itBench({ + id: `${stateId} - capella processHistoricalRootsUpdate`, + beforeEach: () => stateOg.value.clone(), + fn: (state) => processHistoricalRootsUpdate(state, cache.value), + }); + + itBench({ + id: `${stateId} - capella processParticipationFlagUpdates`, + beforeEach: () => stateOg.value.clone() as CachedBeaconStateAltair, + fn: (state) => processParticipationFlagUpdates(state), + }); + + itBench({ + id: `${stateId} - capella afterProcessEpoch`, + // Compute a state and cache after running processEpoch() since those values are mutated + before: () => { + const state = stateOg.value.clone(); + const cacheAfter = beforeProcessEpoch(state); + processEpoch(fork, state, cacheAfter); + return {state, cache: cacheAfter}; + }, + beforeEach: ({state, cache}) => ({state: state.clone(), cache}), + fn: ({state, cache}) => state.epochCtx.afterProcessEpoch(state, cache), + }); +} diff --git a/packages/state-transition/test/perf/params.ts b/packages/state-transition/test/perf/params.ts index 9ab27c62907c..4b6ea2dff90a 100644 --- a/packages/state-transition/test/perf/params.ts +++ b/packages/state-transition/test/perf/params.ts @@ -11,6 +11,11 @@ export const altairState = { epoch: 81889, // Post altair fork }; +export const capellaState = { + network: "mainnet" as const, + epoch: 217614, // Post capella fork +}; + export const rangeSyncTest = { network: "mainnet" as const, startSlot: 3766816, // Post altair, first slot in epoch 117713 diff --git a/packages/state-transition/test/utils/testFileCache.ts b/packages/state-transition/test/utils/testFileCache.ts index cba010bd3738..e752f3c36e68 100644 --- a/packages/state-transition/test/utils/testFileCache.ts +++ b/packages/state-transition/test/utils/testFileCache.ts @@ -104,7 +104,10 @@ async function downloadTestFile(fileId: string): Promise { // eslint-disable-next-line no-console console.log(`Downloading file ${fileUrl}`); - const res = await got(fileUrl, {responseType: "buffer"}); + const res = await got(fileUrl, {responseType: "buffer"}).catch((e: Error) => { + e.message = `Error downloading ${fileUrl}: ${e.message}`; + throw e; + }); return res.body; } From 9afef4190d141a1b9e458fb553899d1425aba7eb Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 27 Jul 2023 14:31:19 -0400 Subject: [PATCH 96/96] v1.10.0 --- lerna.json | 2 +- packages/api/package.json | 10 ++++----- packages/beacon-node/package.json | 26 ++++++++++++------------ packages/cli/package.json | 28 +++++++++++++------------- packages/config/package.json | 6 +++--- packages/db/package.json | 8 ++++---- packages/flare/package.json | 14 ++++++------- packages/fork-choice/package.json | 12 +++++------ packages/light-client/package.json | 14 ++++++------- packages/logger/package.json | 4 ++-- packages/params/package.json | 2 +- packages/prover/package.json | 18 ++++++++--------- packages/reqresp/package.json | 12 +++++------ packages/spec-test-util/package.json | 4 ++-- packages/state-transition/package.json | 10 ++++----- packages/test-utils/package.json | 4 ++-- packages/types/package.json | 4 ++-- packages/utils/package.json | 2 +- packages/validator/package.json | 16 +++++++-------- 19 files changed, 98 insertions(+), 98 deletions(-) diff --git a/lerna.json b/lerna.json index b07dd9032eec..049221287c4e 100644 --- a/lerna.json +++ b/lerna.json @@ -5,7 +5,7 @@ "npmClient": "yarn", "useWorkspaces": true, "useNx": true, - "version": "1.9.2", + "version": "1.10.0", "stream": "true", "command": { "version": { diff --git a/packages/api/package.json b/packages/api/package.json index 3bd8e9d3c47e..527d84a5d108 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.9.2", + "version": "1.10.0", "type": "module", "exports": { ".": { @@ -71,10 +71,10 @@ "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.5.0", "@chainsafe/ssz": "^0.10.2", - "@lodestar/config": "^1.9.2", - "@lodestar/params": "^1.9.2", - "@lodestar/types": "^1.9.2", - "@lodestar/utils": "^1.9.2", + "@lodestar/config": "^1.10.0", + "@lodestar/params": "^1.10.0", + "@lodestar/types": "^1.10.0", + "@lodestar/utils": "^1.10.0", "cross-fetch": "^3.1.8", "eventsource": "^2.0.2", "qs": "^6.11.1" diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 96f37ce38366..2689ae26286c 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.9.2", + "version": "1.10.0", "type": "module", "exports": { ".": { @@ -120,18 +120,18 @@ "@libp2p/peer-id-factory": "^2.0.3", "@libp2p/prometheus-metrics": "^1.1.4", "@libp2p/tcp": "7.0.1", - "@lodestar/api": "^1.9.2", - "@lodestar/config": "^1.9.2", - "@lodestar/db": "^1.9.2", - "@lodestar/fork-choice": "^1.9.2", - "@lodestar/light-client": "^1.9.2", - "@lodestar/logger": "^1.9.2", - "@lodestar/params": "^1.9.2", - "@lodestar/reqresp": "^1.9.2", - "@lodestar/state-transition": "^1.9.2", - "@lodestar/types": "^1.9.2", - "@lodestar/utils": "^1.9.2", - "@lodestar/validator": "^1.9.2", + "@lodestar/api": "^1.10.0", + "@lodestar/config": "^1.10.0", + "@lodestar/db": "^1.10.0", + "@lodestar/fork-choice": "^1.10.0", + "@lodestar/light-client": "^1.10.0", + "@lodestar/logger": "^1.10.0", + "@lodestar/params": "^1.10.0", + "@lodestar/reqresp": "^1.10.0", + "@lodestar/state-transition": "^1.10.0", + "@lodestar/types": "^1.10.0", + "@lodestar/utils": "^1.10.0", + "@lodestar/validator": "^1.10.0", "@multiformats/multiaddr": "^12.1.3", "@types/datastore-level": "^3.0.0", "buffer-xor": "^2.0.2", diff --git a/packages/cli/package.json b/packages/cli/package.json index 29cfbbfba511..180d27a0b9e7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.9.2", + "version": "1.10.0", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -63,17 +63,17 @@ "@libp2p/crypto": "^1.0.0", "@libp2p/peer-id": "^2.0.3", "@libp2p/peer-id-factory": "^2.0.3", - "@lodestar/api": "^1.9.2", - "@lodestar/beacon-node": "^1.9.2", - "@lodestar/config": "^1.9.2", - "@lodestar/db": "^1.9.2", - "@lodestar/light-client": "^1.9.2", - "@lodestar/logger": "^1.9.2", - "@lodestar/params": "^1.9.2", - "@lodestar/state-transition": "^1.9.2", - "@lodestar/types": "^1.9.2", - "@lodestar/utils": "^1.9.2", - "@lodestar/validator": "^1.9.2", + "@lodestar/api": "^1.10.0", + "@lodestar/beacon-node": "^1.10.0", + "@lodestar/config": "^1.10.0", + "@lodestar/db": "^1.10.0", + "@lodestar/light-client": "^1.10.0", + "@lodestar/logger": "^1.10.0", + "@lodestar/params": "^1.10.0", + "@lodestar/state-transition": "^1.10.0", + "@lodestar/types": "^1.10.0", + "@lodestar/utils": "^1.10.0", + "@lodestar/validator": "^1.10.0", "@multiformats/multiaddr": "^12.1.3", "@types/lockfile": "^1.0.2", "bip39": "^3.1.0", @@ -94,12 +94,12 @@ "yargs": "^17.7.1" }, "devDependencies": { + "@lodestar/test-utils": "^1.10.0", "@types/debug": "^4.1.7", "@types/expand-tilde": "^2.0.0", "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", "@types/lodash": "^4.14.192", - "@types/yargs": "^17.0.24", - "@lodestar/test-utils": "^1.9.2" + "@types/yargs": "^17.0.24" } } diff --git a/packages/config/package.json b/packages/config/package.json index de2ca71d577b..6c8ad07f47f5 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.9.2", + "version": "1.10.0", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -65,7 +65,7 @@ ], "dependencies": { "@chainsafe/ssz": "^0.10.2", - "@lodestar/params": "^1.9.2", - "@lodestar/types": "^1.9.2" + "@lodestar/params": "^1.10.0", + "@lodestar/types": "^1.10.0" } } diff --git a/packages/db/package.json b/packages/db/package.json index b2594fd28d44..8018399280ef 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.9.2", + "version": "1.10.0", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", @@ -38,13 +38,13 @@ }, "dependencies": { "@chainsafe/ssz": "^0.10.2", - "@lodestar/config": "^1.9.2", - "@lodestar/utils": "^1.9.2", + "@lodestar/config": "^1.10.0", + "@lodestar/utils": "^1.10.0", "@types/levelup": "^4.3.3", "it-all": "^3.0.2", "level": "^8.0.0" }, "devDependencies": { - "@lodestar/logger": "^1.9.2" + "@lodestar/logger": "^1.10.0" } } diff --git a/packages/flare/package.json b/packages/flare/package.json index 6fc625089b28..96ae5c9f2506 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.9.2", + "version": "1.10.0", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -60,12 +60,12 @@ "dependencies": { "@chainsafe/bls": "7.1.1", "@chainsafe/bls-keygen": "^0.3.0", - "@lodestar/api": "^1.9.2", - "@lodestar/config": "^1.9.2", - "@lodestar/params": "^1.9.2", - "@lodestar/state-transition": "^1.9.2", - "@lodestar/types": "^1.9.2", - "@lodestar/utils": "^1.9.2", + "@lodestar/api": "^1.10.0", + "@lodestar/config": "^1.10.0", + "@lodestar/params": "^1.10.0", + "@lodestar/state-transition": "^1.10.0", + "@lodestar/types": "^1.10.0", + "@lodestar/utils": "^1.10.0", "source-map-support": "^0.5.21", "yargs": "^17.7.1" }, diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index f80dd051a537..060ef5138515 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.9.2", + "version": "1.10.0", "type": "module", "exports": "./lib/index.js", "types": "./lib/index.d.ts", @@ -39,11 +39,11 @@ }, "dependencies": { "@chainsafe/ssz": "^0.10.2", - "@lodestar/config": "^1.9.2", - "@lodestar/params": "^1.9.2", - "@lodestar/state-transition": "^1.9.2", - "@lodestar/types": "^1.9.2", - "@lodestar/utils": "^1.9.2" + "@lodestar/config": "^1.10.0", + "@lodestar/params": "^1.10.0", + "@lodestar/state-transition": "^1.10.0", + "@lodestar/types": "^1.10.0", + "@lodestar/utils": "^1.10.0" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 6be35f6d578f..04e2fc9be907 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.9.2", + "version": "1.10.0", "type": "module", "exports": { ".": { @@ -67,12 +67,12 @@ "@chainsafe/bls": "7.1.1", "@chainsafe/persistent-merkle-tree": "^0.5.0", "@chainsafe/ssz": "^0.10.2", - "@lodestar/api": "^1.9.2", - "@lodestar/config": "^1.9.2", - "@lodestar/params": "^1.9.2", - "@lodestar/state-transition": "^1.9.2", - "@lodestar/types": "^1.9.2", - "@lodestar/utils": "^1.9.2", + "@lodestar/api": "^1.10.0", + "@lodestar/config": "^1.10.0", + "@lodestar/params": "^1.10.0", + "@lodestar/state-transition": "^1.10.0", + "@lodestar/types": "^1.10.0", + "@lodestar/utils": "^1.10.0", "cross-fetch": "^3.1.8", "mitt": "^3.0.0", "strict-event-emitter-types": "^2.0.0" diff --git a/packages/logger/package.json b/packages/logger/package.json index f51202f27b07..a4c918a15ed5 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.9.2", + "version": "1.10.0", "type": "module", "exports": { ".": { @@ -61,7 +61,7 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@lodestar/utils": "^1.9.2", + "@lodestar/utils": "^1.10.0", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1", "winston-transport": "^4.5.0" diff --git a/packages/params/package.json b/packages/params/package.json index 57698dcb008e..7f50080d154a 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.9.2", + "version": "1.10.0", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/prover/package.json b/packages/prover/package.json index 88d0d7d8a317..67af26fb7531 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.9.2", + "version": "1.10.0", "type": "module", "exports": { ".": { @@ -69,13 +69,13 @@ "@ethereumjs/tx": "^4.1.2", "@ethereumjs/util": "^8.0.6", "@ethereumjs/vm": "^6.4.2", - "@lodestar/api": "^1.9.2", - "@lodestar/config": "^1.9.2", - "@lodestar/params": "^1.9.2", - "@lodestar/logger": "^1.9.2", - "@lodestar/light-client": "^1.9.2", - "@lodestar/types": "^1.9.2", - "@lodestar/utils": "^1.9.2", + "@lodestar/api": "^1.10.0", + "@lodestar/config": "^1.10.0", + "@lodestar/light-client": "^1.10.0", + "@lodestar/logger": "^1.10.0", + "@lodestar/params": "^1.10.0", + "@lodestar/types": "^1.10.0", + "@lodestar/utils": "^1.10.0", "ethereum-cryptography": "^1.2.0", "find-up": "^6.3.0", "http-proxy": "^1.18.1", @@ -84,7 +84,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.9.2", + "@lodestar/test-utils": "^1.10.0", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index a8e3c6b84699..cee3e12d90ec 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.9.2", + "version": "1.10.0", "type": "module", "exports": { ".": { @@ -57,9 +57,9 @@ "@chainsafe/fast-crc32c": "^4.1.1", "@libp2p/interface-connection": "^5.1.0", "@libp2p/interface-peer-id": "^2.0.2", - "@lodestar/config": "^1.9.2", - "@lodestar/params": "^1.9.2", - "@lodestar/utils": "^1.9.2", + "@lodestar/config": "^1.10.0", + "@lodestar/params": "^1.10.0", + "@lodestar/utils": "^1.10.0", "it-all": "^3.0.2", "it-pipe": "^3.0.1", "snappy": "^7.2.2", @@ -68,8 +68,8 @@ "varint": "^6.0.0" }, "devDependencies": { - "@lodestar/logger": "^1.9.2", - "@lodestar/types": "^1.9.2", + "@lodestar/logger": "^1.10.0", + "@lodestar/types": "^1.10.0", "libp2p": "0.45.9" }, "peerDependencies": { diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 58409fc8414c..2c78c2d33ea1 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.9.2", + "version": "1.10.0", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -45,7 +45,7 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.9.2", + "@lodestar/utils": "^1.10.0", "async-retry": "^1.3.3", "axios": "^1.3.4", "chai": "^4.3.7", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index c658f22c26ae..1e263f42dd8a 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.9.2", + "version": "1.10.0", "type": "module", "exports": { ".": { @@ -62,10 +62,10 @@ "@chainsafe/persistent-merkle-tree": "^0.5.0", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.10.2", - "@lodestar/config": "^1.9.2", - "@lodestar/params": "^1.9.2", - "@lodestar/types": "^1.9.2", - "@lodestar/utils": "^1.9.2", + "@lodestar/config": "^1.10.0", + "@lodestar/params": "^1.10.0", + "@lodestar/types": "^1.10.0", + "@lodestar/utils": "^1.10.0", "bigint-buffer": "^1.1.5", "buffer-xor": "^2.0.2" }, diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 78caa24cc6e4..975a82a38d8d 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.9.2", + "version": "1.10.0", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -61,7 +61,7 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.9.2", + "@lodestar/utils": "^1.10.0", "axios": "^1.3.4", "chai": "^4.3.7", "mocha": "^10.2.0", diff --git a/packages/types/package.json b/packages/types/package.json index b5203891cf96..522b1a56ab27 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.9.2", + "version": "1.10.0", "type": "module", "exports": { ".": { @@ -68,7 +68,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/ssz": "^0.10.2", - "@lodestar/params": "^1.9.2" + "@lodestar/params": "^1.10.0" }, "keywords": [ "ethereum", diff --git a/packages/utils/package.json b/packages/utils/package.json index b943ab62edb4..edb28f92508b 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.9.2", + "version": "1.10.0", "type": "module", "exports": "./lib/index.js", "files": [ diff --git a/packages/validator/package.json b/packages/validator/package.json index c199ceb2dbca..d312505fcf71 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.9.2", + "version": "1.10.0", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -50,13 +50,13 @@ "dependencies": { "@chainsafe/bls": "7.1.1", "@chainsafe/ssz": "^0.10.2", - "@lodestar/api": "^1.9.2", - "@lodestar/config": "^1.9.2", - "@lodestar/db": "^1.9.2", - "@lodestar/params": "^1.9.2", - "@lodestar/state-transition": "^1.9.2", - "@lodestar/types": "^1.9.2", - "@lodestar/utils": "^1.9.2", + "@lodestar/api": "^1.10.0", + "@lodestar/config": "^1.10.0", + "@lodestar/db": "^1.10.0", + "@lodestar/params": "^1.10.0", + "@lodestar/state-transition": "^1.10.0", + "@lodestar/types": "^1.10.0", + "@lodestar/utils": "^1.10.0", "bigint-buffer": "^1.1.5", "cross-fetch": "^3.1.8", "strict-event-emitter-types": "^2.0.0"