From 0c351d787222eb9b4d75bdff86ccc589e462c950 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Mon, 24 Jul 2023 15:18:28 +0700 Subject: [PATCH 1/6] feat: poll proposer duties of next epoch in advance --- .../src/api/impl/validator/index.ts | 43 ++++++++++++++----- .../beacon-node/src/chain/prepareNextSlot.ts | 2 +- .../src/metrics/metrics/lodestar.ts | 11 +++++ packages/validator/src/services/block.ts | 1 + .../validator/src/services/blockDuties.ts | 31 ++++++++++--- .../test/unit/services/blockDuties.test.ts | 31 +++++++++++-- 6 files changed, 100 insertions(+), 19 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index bc0b25a953d6..3b9fa0332419 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -32,6 +32,7 @@ import {ApiModules} from "../types.js"; import {RegenCaller} from "../../../chain/regen/index.js"; import {getValidatorStatus} from "../beacon/state/utils.js"; import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js"; +import {SCHEDULER_LOOKAHEAD_FACTOR} from "../../../chain/prepareNextSlot.js"; import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices} from "./utils.js"; /** @@ -118,15 +119,23 @@ export function getValidatorApi({ * Prevents a validator from not being able to get the attestater duties correctly if the beacon and validator clocks are off */ async function waitForNextClosestEpoch(): Promise { - const nextEpoch = chain.clock.currentEpoch + 1; - const secPerEpoch = SLOTS_PER_EPOCH * config.SECONDS_PER_SLOT; - const nextEpochStartSec = chain.genesisTime + nextEpoch * secPerEpoch; - const msToNextEpoch = nextEpochStartSec * 1000 - Date.now(); + const msToNextEpoch = timeToNextEpochInMs(); if (msToNextEpoch > 0 && msToNextEpoch < MAX_API_CLOCK_DISPARITY_MS) { + const nextEpoch = chain.clock.currentEpoch + 1; await chain.clock.waitForSlot(computeStartSlotAtEpoch(nextEpoch)); } } + /** + * Compute ms to the next epoch. + */ + function timeToNextEpochInMs(): number { + const nextEpoch = chain.clock.currentEpoch + 1; + const secPerEpoch = SLOTS_PER_EPOCH * config.SECONDS_PER_SLOT; + const nextEpochStartSec = chain.genesisTime + nextEpoch * secPerEpoch; + return nextEpochStartSec * 1000 - Date.now(); + } + function currentEpochWithDisparity(): Epoch { return computeEpochAtSlot(getCurrentSlot(config, chain.genesisTime - MAX_API_CLOCK_DISPARITY_SEC)); } @@ -387,15 +396,29 @@ export function getValidatorApi({ // Early check that epoch is within [current_epoch, current_epoch + 1], or allow for pre-genesis const currentEpoch = currentEpochWithDisparity(); - if (currentEpoch >= 0 && epoch !== currentEpoch && epoch !== currentEpoch + 1) { - throw Error(`Requested epoch ${epoch} must equal current ${currentEpoch} or next epoch ${currentEpoch + 1}`); + const nextEpoch = currentEpoch + 1; + if (currentEpoch >= 0 && epoch !== currentEpoch && epoch !== nextEpoch) { + throw Error(`Requested epoch ${epoch} must equal current ${currentEpoch} or next epoch ${nextEpoch}`); } - // May request for an epoch that's in the future, for getBeaconProposersNextEpoch() - await waitForNextClosestEpoch(); - const head = chain.forkChoice.getHead(); - const state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties); + let state: CachedBeaconStateAllForks | undefined = undefined; + const prepareNextSlotLookAheadMs = (config.SECONDS_PER_SLOT * 1000) / SCHEDULER_LOOKAHEAD_FACTOR; + const cpState = chain.regen.getCheckpointStateSync({rootHex: head.blockRoot, epoch}); + // validators may request next epoch's duties when it's close to next epoch + // return that asap if PrepareNextSlot already compute beacon proposers for next epoch + if (epoch === nextEpoch && timeToNextEpochInMs() < prepareNextSlotLookAheadMs) { + if (cpState) { + state = cpState; + metrics?.duties.requestNextEpochProposalDutiesHit.inc(); + } else { + metrics?.duties.requestNextEpochProposalDutiesMiss.inc(); + } + } + + if (!state) { + state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties); + } const stateEpoch = state.epochCtx.epoch; let indexes: ValidatorIndex[] = []; diff --git a/packages/beacon-node/src/chain/prepareNextSlot.ts b/packages/beacon-node/src/chain/prepareNextSlot.ts index 38a97efd38a2..1091fd716b60 100644 --- a/packages/beacon-node/src/chain/prepareNextSlot.ts +++ b/packages/beacon-node/src/chain/prepareNextSlot.ts @@ -14,7 +14,7 @@ import {IBeaconChain} from "./interface.js"; import {RegenCaller} from "./regen/index.js"; /* With 12s slot times, this scheduler will run 4s before the start of each slot (`12 / 3 = 4`). */ -const SCHEDULER_LOOKAHEAD_FACTOR = 3; +export const SCHEDULER_LOOKAHEAD_FACTOR = 3; /* We don't want to do more epoch transition than this */ const PREPARE_EPOCH_LIMIT = 1; diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 04915a064899..8b3410a6eadf 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -245,6 +245,17 @@ export function createLodestarMetrics( }), }, + duties: { + requestNextEpochProposalDutiesHit: register.gauge({ + name: "lodestar_duties_request_next_epoch_proposal_duties_hit_total", + help: "Total count of requestNextEpochProposalDuties hit", + }), + requestNextEpochProposalDutiesMiss: register.gauge({ + name: "lodestar_duties_request_next_epoch_proposal_duties_miss_total", + help: "Total count of requestNextEpochProposalDuties miss", + }), + }, + // Beacon state transition metrics epochTransitionTime: register.histogram({ diff --git a/packages/validator/src/services/block.ts b/packages/validator/src/services/block.ts index 2627bb1892e1..c1385fce9a9c 100644 --- a/packages/validator/src/services/block.ts +++ b/packages/validator/src/services/block.ts @@ -69,6 +69,7 @@ export class BlockProposingService { private readonly metrics: Metrics | null ) { this.dutiesService = new BlockDutiesService( + config, logger, api, clock, diff --git a/packages/validator/src/services/blockDuties.ts b/packages/validator/src/services/blockDuties.ts index d0812f173c9e..c6064203044e 100644 --- a/packages/validator/src/services/blockDuties.ts +++ b/packages/validator/src/services/blockDuties.ts @@ -1,12 +1,19 @@ import {toHexString} from "@chainsafe/ssz"; -import {computeEpochAtSlot} from "@lodestar/state-transition"; +import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {BLSPubkey, Epoch, RootHex, Slot} from "@lodestar/types"; import {Api, ApiError, routes} from "@lodestar/api"; +import {sleep} from "@lodestar/utils"; +import {ChainConfig} from "@lodestar/config"; import {IClock, differenceHex, LoggerVc} from "../util/index.js"; import {PubkeyHex} from "../types.js"; import {Metrics} from "../metrics.js"; import {ValidatorStore} from "./validatorStore.js"; +/** This polls block duties 1s before the next epoch */ +// TODO: change to 6 to do it 2s before the next epoch +// once we have some improvement on epoch transition time +// see https://github.com/ChainSafe/lodestar/issues/5409 +const BLOCK_DUTIES_LOOKAHEAD_FACTOR = 12; /** Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch */ const HISTORICAL_DUTIES_EPOCHS = 2; // Re-declaring to not have to depend on `lodestar-params` just for this 0 @@ -24,9 +31,10 @@ export class BlockDutiesService { private readonly proposers = new Map(); constructor( + private readonly config: ChainConfig, private readonly logger: LoggerVc, private readonly api: Api, - clock: IClock, + private readonly clock: IClock, private readonly validatorStore: ValidatorStore, private readonly metrics: Metrics | null, notifyBlockProductionFn: NotifyBlockProductionFn @@ -75,7 +83,7 @@ export class BlockDutiesService { } } - private runBlockDutiesTask = async (slot: Slot): Promise => { + private runBlockDutiesTask = async (slot: Slot, signal: AbortSignal): Promise => { try { if (slot < 0) { // Before genesis, fetch the genesis duties but don't notify block production @@ -84,7 +92,7 @@ export class BlockDutiesService { await this.pollBeaconProposers(GENESIS_EPOCH); } } else { - await this.pollBeaconProposersAndNotify(slot); + await this.pollBeaconProposersAndNotify(slot, signal); } } catch (e) { this.logger.error("Error on pollBeaconProposers", {}, e as Error); @@ -117,8 +125,10 @@ export class BlockDutiesService { * through the slow path every time. I.e., the proposal will only happen after we've been able to * download and process the duties from the BN. This means it is very important to ensure this * function is as fast as possible. + * - Starting from Jul 2023, if PrepareNextSlotScheduler runs well in bn we already have proposers of next epoch + * some time (< 1/3 slot) before the next epoch */ - private async pollBeaconProposersAndNotify(currentSlot: Slot): Promise { + private async pollBeaconProposersAndNotify(currentSlot: Slot, signal: AbortSignal): Promise { // Notify the block proposal service for any proposals that we have in our cache. const initialBlockProposers = this.getblockProposersAtSlot(currentSlot); if (initialBlockProposers.length > 0) { @@ -143,6 +153,17 @@ export class BlockDutiesService { this.logger.debug("Detected new block proposer", {currentSlot}); this.metrics?.proposerDutiesReorg.inc(); } + + const nextEpoch = computeEpochAtSlot(currentSlot) + 1; + const isLastSlotEpoch = computeStartSlotAtEpoch(nextEpoch) === currentSlot + 1; + if (isLastSlotEpoch) { + const nextSlot = currentSlot + 1; + const lookAheadMs = (this.config.SECONDS_PER_SLOT * 1000) / BLOCK_DUTIES_LOOKAHEAD_FACTOR; + await sleep(this.clock.msToSlot(nextSlot) - lookAheadMs, signal); + this.logger.debug("Polling proposers for next epoch", {nextEpoch, nextSlot}); + // Poll proposers for the next epoch + await this.pollBeaconProposers(nextEpoch); + } } private async pollBeaconProposers(epoch: Epoch): Promise { diff --git a/packages/validator/test/unit/services/blockDuties.test.ts b/packages/validator/test/unit/services/blockDuties.test.ts index fa22984bc59a..d6152f6d7b09 100644 --- a/packages/validator/test/unit/services/blockDuties.test.ts +++ b/packages/validator/test/unit/services/blockDuties.test.ts @@ -5,6 +5,7 @@ import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; import {RootHex} from "@lodestar/types"; import {HttpStatusCode, routes} from "@lodestar/api"; +import {chainConfig} from "@lodestar/config/default"; import {toHex} from "@lodestar/utils"; import {BlockDutiesService} from "../../../src/services/blockDuties.js"; import {ValidatorStore} from "../../../src/services/validatorStore.js"; @@ -49,7 +50,15 @@ describe("BlockDutiesService", function () { const notifyBlockProductionFn = sinon.stub(); // Returns void const clock = new ClockMock(); - const dutiesService = new BlockDutiesService(loggerVc, api, clock, validatorStore, null, notifyBlockProductionFn); + const dutiesService = new BlockDutiesService( + chainConfig, + loggerVc, + api, + clock, + validatorStore, + null, + notifyBlockProductionFn + ); // Trigger clock onSlot for slot 0 await clock.tickSlotFns(0, controller.signal); @@ -84,7 +93,15 @@ describe("BlockDutiesService", function () { // Clock will call runAttesterDutiesTasks() immediately const clock = new ClockMock(); - const dutiesService = new BlockDutiesService(loggerVc, api, clock, validatorStore, null, notifyBlockProductionFn); + const dutiesService = new BlockDutiesService( + chainConfig, + loggerVc, + api, + clock, + validatorStore, + null, + notifyBlockProductionFn + ); // Trigger clock onSlot for slot 0 api.validator.getProposerDuties.resolves({ @@ -151,7 +168,15 @@ describe("BlockDutiesService", function () { const notifyBlockProductionFn = sinon.stub(); // Returns void const clock = new ClockMock(); - const dutiesService = new BlockDutiesService(loggerVc, api, clock, validatorStore, null, notifyBlockProductionFn); + const dutiesService = new BlockDutiesService( + chainConfig, + loggerVc, + api, + clock, + validatorStore, + null, + notifyBlockProductionFn + ); // Trigger clock onSlot for slot 0 await clock.tickSlotFns(0, controller.signal); From 96fe621fae89c0c7bcc620db2ebc08510185ef22 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Mon, 24 Jul 2023 19:54:19 +0700 Subject: [PATCH 2/6] chore: refactor function name msToNextEpoch --- packages/beacon-node/src/api/impl/validator/index.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 3b9fa0332419..d988af66a691 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -119,8 +119,8 @@ export function getValidatorApi({ * Prevents a validator from not being able to get the attestater duties correctly if the beacon and validator clocks are off */ async function waitForNextClosestEpoch(): Promise { - const msToNextEpoch = timeToNextEpochInMs(); - if (msToNextEpoch > 0 && msToNextEpoch < MAX_API_CLOCK_DISPARITY_MS) { + const toNextEpochMs = msToNextEpoch(); + if (toNextEpochMs > 0 && toNextEpochMs < MAX_API_CLOCK_DISPARITY_MS) { const nextEpoch = chain.clock.currentEpoch + 1; await chain.clock.waitForSlot(computeStartSlotAtEpoch(nextEpoch)); } @@ -129,7 +129,7 @@ export function getValidatorApi({ /** * Compute ms to the next epoch. */ - function timeToNextEpochInMs(): number { + function msToNextEpoch(): number { const nextEpoch = chain.clock.currentEpoch + 1; const secPerEpoch = SLOTS_PER_EPOCH * config.SECONDS_PER_SLOT; const nextEpochStartSec = chain.genesisTime + nextEpoch * secPerEpoch; @@ -405,9 +405,10 @@ export function getValidatorApi({ let state: CachedBeaconStateAllForks | undefined = undefined; const prepareNextSlotLookAheadMs = (config.SECONDS_PER_SLOT * 1000) / SCHEDULER_LOOKAHEAD_FACTOR; const cpState = chain.regen.getCheckpointStateSync({rootHex: head.blockRoot, epoch}); + const toNextEpochMs = msToNextEpoch(); // validators may request next epoch's duties when it's close to next epoch // return that asap if PrepareNextSlot already compute beacon proposers for next epoch - if (epoch === nextEpoch && timeToNextEpochInMs() < prepareNextSlotLookAheadMs) { + if (epoch === nextEpoch && toNextEpochMs < prepareNextSlotLookAheadMs) { if (cpState) { state = cpState; metrics?.duties.requestNextEpochProposalDutiesHit.inc(); From 10c5c22e31b288fe57a2c20b43376cae06e02751 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Mon, 24 Jul 2023 20:28:32 +0700 Subject: [PATCH 3/6] fix: pollBeaconProposersNextEpoch --- .../validator/src/services/blockDuties.ts | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/validator/src/services/blockDuties.ts b/packages/validator/src/services/blockDuties.ts index c6064203044e..a1a364712fdf 100644 --- a/packages/validator/src/services/blockDuties.ts +++ b/packages/validator/src/services/blockDuties.ts @@ -129,6 +129,13 @@ export class BlockDutiesService { * some time (< 1/3 slot) before the next epoch */ private async pollBeaconProposersAndNotify(currentSlot: Slot, signal: AbortSignal): Promise { + const nextEpoch = computeEpochAtSlot(currentSlot) + 1; + const isLastSlotEpoch = computeStartSlotAtEpoch(nextEpoch) === currentSlot + 1; + if (isLastSlotEpoch) { + // do this at the end of the function would be too late + void this.pollBeaconProposersNextEpoch(currentSlot, nextEpoch, signal); + } + // Notify the block proposal service for any proposals that we have in our cache. const initialBlockProposers = this.getblockProposersAtSlot(currentSlot); if (initialBlockProposers.length > 0) { @@ -153,17 +160,15 @@ export class BlockDutiesService { this.logger.debug("Detected new block proposer", {currentSlot}); this.metrics?.proposerDutiesReorg.inc(); } + } - const nextEpoch = computeEpochAtSlot(currentSlot) + 1; - const isLastSlotEpoch = computeStartSlotAtEpoch(nextEpoch) === currentSlot + 1; - if (isLastSlotEpoch) { - const nextSlot = currentSlot + 1; - const lookAheadMs = (this.config.SECONDS_PER_SLOT * 1000) / BLOCK_DUTIES_LOOKAHEAD_FACTOR; - await sleep(this.clock.msToSlot(nextSlot) - lookAheadMs, signal); - this.logger.debug("Polling proposers for next epoch", {nextEpoch, nextSlot}); - // Poll proposers for the next epoch - await this.pollBeaconProposers(nextEpoch); - } + private async pollBeaconProposersNextEpoch(currentSlot: Slot, nextEpoch: Epoch, signal: AbortSignal): Promise { + const nextSlot = currentSlot + 1; + const lookAheadMs = (this.config.SECONDS_PER_SLOT * 1000) / BLOCK_DUTIES_LOOKAHEAD_FACTOR; + await sleep(this.clock.msToSlot(nextSlot) - lookAheadMs, signal); + this.logger.debug("Polling proposers for next epoch", {nextEpoch, nextSlot}); + // Poll proposers for the next epoch + await this.pollBeaconProposers(nextEpoch); } private async pollBeaconProposers(epoch: Epoch): Promise { From e33230035862ee18e8634ca176d2ea7e087b70a8 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Mon, 24 Jul 2023 20:56:44 +0700 Subject: [PATCH 4/6] fix: set MAX_API_CLOCK_DISPARITY_SEC to 0.5s --- packages/beacon-node/src/api/impl/validator/index.ts | 3 ++- packages/validator/src/services/blockDuties.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index d988af66a691..99dd03763ab4 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -67,9 +67,10 @@ export function getValidatorApi({ /** * Validator clock may be advanced from beacon's clock. If the validator requests a resource in a * future slot, wait some time instead of rejecting the request because it's in the future. + * This value is the same to MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC. * For very fast networks, reduce clock disparity to half a slot. */ - const MAX_API_CLOCK_DISPARITY_SEC = Math.min(1, config.SECONDS_PER_SLOT / 2); + const MAX_API_CLOCK_DISPARITY_SEC = Math.min(0.5, config.SECONDS_PER_SLOT / 2); const MAX_API_CLOCK_DISPARITY_MS = MAX_API_CLOCK_DISPARITY_SEC * 1000; /** Compute and cache the genesis block root */ diff --git a/packages/validator/src/services/blockDuties.ts b/packages/validator/src/services/blockDuties.ts index a1a364712fdf..a6bf0de3088f 100644 --- a/packages/validator/src/services/blockDuties.ts +++ b/packages/validator/src/services/blockDuties.ts @@ -132,7 +132,7 @@ export class BlockDutiesService { const nextEpoch = computeEpochAtSlot(currentSlot) + 1; const isLastSlotEpoch = computeStartSlotAtEpoch(nextEpoch) === currentSlot + 1; if (isLastSlotEpoch) { - // do this at the end of the function would be too late + // no need to await for other steps, just poll proposers for next epoch void this.pollBeaconProposersNextEpoch(currentSlot, nextEpoch, signal); } From da3048709e3ee0b3ee35c056bd9c5dae15ab806a Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 25 Jul 2023 10:13:20 +0700 Subject: [PATCH 5/6] chore: add more comments --- packages/beacon-node/src/api/impl/validator/index.ts | 4 ++-- packages/validator/src/services/blockDuties.ts | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 99dd03763ab4..27efa389a11e 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -405,11 +405,11 @@ export function getValidatorApi({ const head = chain.forkChoice.getHead(); let state: CachedBeaconStateAllForks | undefined = undefined; const prepareNextSlotLookAheadMs = (config.SECONDS_PER_SLOT * 1000) / SCHEDULER_LOOKAHEAD_FACTOR; - const cpState = chain.regen.getCheckpointStateSync({rootHex: head.blockRoot, epoch}); const toNextEpochMs = msToNextEpoch(); // validators may request next epoch's duties when it's close to next epoch - // return that asap if PrepareNextSlot already compute beacon proposers for next epoch + // this is to avoid missed block proposal due to 0 epoch look ahead if (epoch === nextEpoch && toNextEpochMs < prepareNextSlotLookAheadMs) { + const cpState = chain.regen.getCheckpointStateSync({rootHex: head.blockRoot, epoch}); if (cpState) { state = cpState; metrics?.duties.requestNextEpochProposalDutiesHit.inc(); diff --git a/packages/validator/src/services/blockDuties.ts b/packages/validator/src/services/blockDuties.ts index a6bf0de3088f..65b98ce3b1f0 100644 --- a/packages/validator/src/services/blockDuties.ts +++ b/packages/validator/src/services/blockDuties.ts @@ -12,7 +12,7 @@ import {ValidatorStore} from "./validatorStore.js"; /** This polls block duties 1s before the next epoch */ // TODO: change to 6 to do it 2s before the next epoch // once we have some improvement on epoch transition time -// see https://github.com/ChainSafe/lodestar/issues/5409 +// see https://github.com/ChainSafe/lodestar/issues/5792#issuecomment-1647457442 const BLOCK_DUTIES_LOOKAHEAD_FACTOR = 12; /** Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch */ const HISTORICAL_DUTIES_EPOCHS = 2; @@ -125,8 +125,8 @@ export class BlockDutiesService { * through the slow path every time. I.e., the proposal will only happen after we've been able to * download and process the duties from the BN. This means it is very important to ensure this * function is as fast as possible. - * - Starting from Jul 2023, if PrepareNextSlotScheduler runs well in bn we already have proposers of next epoch - * some time (< 1/3 slot) before the next epoch + * - Starting from Jul 2023, we poll proposers 1s before the next epoch thanks to PrepareNextSlotScheduler + * usually finishes in 3s. */ private async pollBeaconProposersAndNotify(currentSlot: Slot, signal: AbortSignal): Promise { const nextEpoch = computeEpochAtSlot(currentSlot) + 1; @@ -162,6 +162,10 @@ export class BlockDutiesService { } } + /** + * This is to avoid some delay on the first slot of the opoch when validators has proposal duties. + * See https://github.com/ChainSafe/lodestar/issues/5792 + */ private async pollBeaconProposersNextEpoch(currentSlot: Slot, nextEpoch: Epoch, signal: AbortSignal): Promise { const nextSlot = currentSlot + 1; const lookAheadMs = (this.config.SECONDS_PER_SLOT * 1000) / BLOCK_DUTIES_LOOKAHEAD_FACTOR; From c08ac108638d0140ec063a1f647785a18a9adbb8 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 25 Jul 2023 10:57:23 +0700 Subject: [PATCH 6/6] feat: wait for checkpoint state up to 1 slot time --- .../src/api/impl/validator/index.ts | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 27efa389a11e..d8d405f334f0 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -33,6 +33,7 @@ import {RegenCaller} from "../../../chain/regen/index.js"; import {getValidatorStatus} from "../beacon/state/utils.js"; import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js"; import {SCHEDULER_LOOKAHEAD_FACTOR} from "../../../chain/prepareNextSlot.js"; +import {ChainEvent, CheckpointHex} from "../../../chain/index.js"; import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices} from "./utils.js"; /** @@ -141,6 +142,34 @@ export function getValidatorApi({ return computeEpochAtSlot(getCurrentSlot(config, chain.genesisTime - MAX_API_CLOCK_DISPARITY_SEC)); } + /** + * This function is called 1s before next epoch, usually at that time PrepareNextSlotScheduler finishes + * so we should have checkpoint state, otherwise wait for up to `timeoutMs`. + */ + async function waitForCheckpointState( + cpHex: CheckpointHex, + timeoutMs: number + ): Promise { + const cpState = chain.regen.getCheckpointStateSync(cpHex); + if (cpState) { + return cpState; + } + const cp = { + epoch: cpHex.epoch, + root: fromHexString(cpHex.rootHex), + }; + // if not, wait for ChainEvent.checkpoint event until timeoutMs + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(null), timeoutMs); + chain.emitter.on(ChainEvent.checkpoint, (eventCp, cpState) => { + if (ssz.phase0.Checkpoint.equals(eventCp, cp)) { + clearTimeout(timer); + resolve(cpState); + } + }); + }); + } + /** * Reject any request while the node is syncing */ @@ -404,12 +433,14 @@ export function getValidatorApi({ const head = chain.forkChoice.getHead(); let state: CachedBeaconStateAllForks | undefined = undefined; - const prepareNextSlotLookAheadMs = (config.SECONDS_PER_SLOT * 1000) / SCHEDULER_LOOKAHEAD_FACTOR; + const slotMs = config.SECONDS_PER_SLOT * 1000; + const prepareNextSlotLookAheadMs = slotMs / SCHEDULER_LOOKAHEAD_FACTOR; const toNextEpochMs = msToNextEpoch(); // validators may request next epoch's duties when it's close to next epoch // this is to avoid missed block proposal due to 0 epoch look ahead if (epoch === nextEpoch && toNextEpochMs < prepareNextSlotLookAheadMs) { - const cpState = chain.regen.getCheckpointStateSync({rootHex: head.blockRoot, epoch}); + // wait for maximum 1 slot for cp state which is the timeout of validator api + const cpState = await waitForCheckpointState({rootHex: head.blockRoot, epoch}, slotMs); if (cpState) { state = cpState; metrics?.duties.requestNextEpochProposalDutiesHit.inc();