diff --git a/packages/beacon-node/src/chain/archiver/index.ts b/packages/beacon-node/src/chain/archiver/index.ts index 294c2281e19b..e2f55b8f0500 100644 --- a/packages/beacon-node/src/chain/archiver/index.ts +++ b/packages/beacon-node/src/chain/archiver/index.ts @@ -107,7 +107,7 @@ export class Archiver { // should be after ArchiveBlocksTask to handle restart cleanly await this.statesArchiver.maybeArchiveState(finalized); - this.chain.regen.pruneOnFinalized(finalizedEpoch); + this.chain.pruneOnFinalized(finalizedEpoch); // tasks rely on extended fork choice const prunedBlocks = this.chain.forkChoice.prune(finalized.rootHex); diff --git a/packages/beacon-node/src/chain/balancesTreeCache.ts b/packages/beacon-node/src/chain/balancesTreeCache.ts new file mode 100644 index 000000000000..462ae860809e --- /dev/null +++ b/packages/beacon-node/src/chain/balancesTreeCache.ts @@ -0,0 +1,38 @@ +import {ListBasicTreeViewDU, UintNumberType} from "@chainsafe/ssz"; +import {IBalancesTreeCache, CachedBeaconStateAllForks} from "@lodestar/state-transition"; +import {Metrics} from "../metrics/index.js"; + +const MAX_ITEMS = 2; + +export class BalancesTreeCache implements IBalancesTreeCache { + private readonly unusedBalancesTrees: ListBasicTreeViewDU[] = []; + + constructor(private readonly metrics: Metrics | null = null) { + if (metrics) { + metrics.balancesTreeCache.size.addCollect(() => { + metrics.balancesTreeCache.size.set(this.unusedBalancesTrees.length); + }); + } + } + + processUnusedState(state: CachedBeaconStateAllForks | undefined): void { + if (state === undefined) { + return; + } + + this.unusedBalancesTrees.push(state.balances); + while (this.unusedBalancesTrees.length > MAX_ITEMS) { + this.unusedBalancesTrees.shift(); + } + } + + getUnusedBalances(): ListBasicTreeViewDU | undefined { + if (this.unusedBalancesTrees.length === 0) { + this.metrics?.balancesTreeCache.miss.inc(); + return undefined; + } + + this.metrics?.balancesTreeCache.hit.inc(); + return this.unusedBalancesTrees.shift(); + } +} diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index de5ecf607d95..1d1772a1692f 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -97,7 +97,16 @@ export async function importBlock( // This adds the state necessary to process the next block // Some block event handlers require state being in state cache so need to do this before emitting EventType.block - this.regen.processState(blockRootHex, postState); + this.regen.processState(blockRootHex, postState).then((prunedStates) => { + if (prunedStates) { + for (const states of prunedStates.values()) { + // cp states on the same epoch shares the same balances seed tree so only need one of them + this.balancesTreeCache.processUnusedState(states[0]); + } + } + }).catch((e) => { + this.logger.error("Regen error to process state for block", {slot: blockSlot, root: blockRootHex}, e as Error); + }); this.metrics?.importBlock.bySource.inc({source}); this.logger.verbose("Added block to forkchoice and state cache", {slot: blockSlot, root: blockRootHex}); diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 8dbb49798538..69d6a6b41c56 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -101,6 +101,7 @@ import {DbCPStateDatastore} from "./stateCache/datastore/db.js"; import {FileCPStateDatastore} from "./stateCache/datastore/file.js"; import {SyncCommitteeRewards, computeSyncCommitteeRewards} from "./rewards/syncCommitteeRewards.js"; import {AttestationsRewards, computeAttestationsRewards} from "./rewards/attestationsRewards.js"; +import {BalancesTreeCache} from "./balancesTreeCache.js"; /** * Arbitrary constants, blobs and payloads should be consumed immediately in the same slot @@ -158,6 +159,7 @@ export class BeaconChain implements IBeaconChain { readonly beaconProposerCache: BeaconProposerCache; readonly checkpointBalancesCache: CheckpointBalancesCache; readonly shufflingCache: ShufflingCache; + readonly balancesTreeCache: BalancesTreeCache; /** Map keyed by executionPayload.blockHash of the block for those blobs */ readonly producedContentsCache = new Map(); @@ -247,6 +249,7 @@ export class BeaconChain implements IBeaconChain { this.beaconProposerCache = new BeaconProposerCache(opts); this.checkpointBalancesCache = new CheckpointBalancesCache(); this.shufflingCache = new ShufflingCache(metrics, this.opts); + this.balancesTreeCache = new BalancesTreeCache(metrics); // Restore state caches // anchorState may already by a CachedBeaconState. If so, don't create the cache again, since deserializing all @@ -260,6 +263,7 @@ export class BeaconChain implements IBeaconChain { config, pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + balancesTreeCache: this.balancesTreeCache, }); this.shufflingCache.processState(cachedState, cachedState.epochCtx.previousShuffling.epoch); this.shufflingCache.processState(cachedState, cachedState.epochCtx.currentShuffling.epoch); @@ -863,6 +867,16 @@ export class BeaconChain implements IBeaconChain { } } + pruneOnFinalized(finalizedEpoch: Epoch): void { + const prunedStates = this.regen.pruneOnFinalized(finalizedEpoch); + if (prunedStates) { + // cp states on the same epoch shares the same balances seed tree so only need one of them + for (const states of prunedStates.values()) { + this.balancesTreeCache.processUnusedState(states[0]); + } + } + } + /** * Regenerate state for attestation verification, this does not happen with default chain option of maxSkipSlots = 32 . * However, need to handle just in case. Lodestar doesn't support multiple regen state requests for attestation verification diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 5185662eaa4f..e70d5a0c8297 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -241,6 +241,8 @@ export interface IBeaconChain { blockRef: BeaconBlock | BlindedBeaconBlock, validatorIds?: (ValidatorIndex | string)[] ): Promise; + + pruneOnFinalized(finalizedEpoch: Epoch): void; } export type SSZObjectType = diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 57e64bd364ea..45b8b4f76c5f 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -148,16 +148,26 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.blockStateCache.prune(headStateRoot); } - pruneOnFinalized(finalizedEpoch: number): void { - this.checkpointStateCache.pruneFinalized(finalizedEpoch); + pruneOnFinalized(finalizedEpoch: number): Map | null { + const prunedStates = this.checkpointStateCache.pruneFinalized(finalizedEpoch); this.blockStateCache.deleteAllBeforeEpoch(finalizedEpoch); + + return prunedStates; } - processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void { + async processState( + blockRootHex: RootHex, + postState: CachedBeaconStateAllForks + ): Promise | null> { this.blockStateCache.add(postState); - this.checkpointStateCache.processState(blockRootHex, postState).catch((e) => { - this.logger.debug("Error processing block state", {blockRootHex, slot: postState.slot}, e); - }); + let prunedStates: Map | null = null; + try { + prunedStates = await this.checkpointStateCache.processState(blockRootHex, postState); + } catch (e) { + this.logger.debug("Error processing block state", {blockRootHex, slot: postState.slot}, e as Error); + } + + return prunedStates; } addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void { diff --git a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts index 38aeabb97955..bb1ff18e25de 100644 --- a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts @@ -59,9 +59,9 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { return this.getLatest(rootHex, maxEpoch, opts); } - async processState(): Promise { + async processState(): Promise | null> { // do nothing, this class does not support prunning - return 0; + return null; } get(cp: CheckpointHex, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { @@ -122,12 +122,17 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { return previousHits; } - pruneFinalized(finalizedEpoch: Epoch): void { + pruneFinalized(finalizedEpoch: Epoch): Map { + const result = new Map(); + for (const epoch of this.epochIndex.keys()) { if (epoch < finalizedEpoch) { - this.deleteAllEpochItems(epoch); + const deletedStates = this.deleteAllEpochItems(epoch); + result.set(epoch, deletedStates); } } + + return result; } prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void { @@ -153,11 +158,19 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { } } - deleteAllEpochItems(epoch: Epoch): void { + deleteAllEpochItems(epoch: Epoch): CachedBeaconStateAllForks[] { + const states = []; for (const rootHex of this.epochIndex.get(epoch) || []) { - this.cache.delete(toCheckpointKey({rootHex, epoch})); + const key = toCheckpointKey({rootHex, epoch}); + const state = this.cache.get(key); + if (state) { + states.push(state); + } + this.cache.delete(key); } this.epochIndex.delete(epoch); + + return states; } clear(): void { diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 190b79e58cd6..b4d16ae1d500 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -421,7 +421,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Prune all checkpoint states before the provided finalized epoch. */ - pruneFinalized(finalizedEpoch: Epoch): void { + pruneFinalized(finalizedEpoch: Epoch): Map | null { for (const epoch of this.epochIndex.keys()) { if (epoch < finalizedEpoch) { this.deleteAllEpochItems(epoch).catch((e) => @@ -429,6 +429,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { ); } } + + // not likely to return anything in-memory state because we may persist states even before they are finalized + return null; } /** @@ -481,12 +484,14 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * * As of Mar 2024, it takes <=350ms to persist a holesky state on fast server */ - async processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise { - let persistCount = 0; + async processState( + blockRootHex: RootHex, + state: CachedBeaconStateAllForks + ): Promise | null> { // it's important to sort the epochs in ascending order, in case of big reorg we always want to keep the most recent checkpoint states const sortedEpochs = Array.from(this.epochIndex.keys()).sort((a, b) => a - b); if (sortedEpochs.length <= this.maxEpochsInMemory) { - return 0; + return null; } const blockSlot = state.slot; @@ -502,24 +507,19 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { // normally the block persist happens at 2/3 of slot 0 of epoch, if it's already late then just skip to allow other tasks to run // there are plenty of chances in the same epoch to persist checkpoint states, also if block is late it could be reorged this.logger.verbose("Skip persist checkpoint states", {blockSlot, root: blockRootHex}); - return 0; + return null; } const persistEpochs = sortedEpochs.slice(0, sortedEpochs.length - this.maxEpochsInMemory); + + const result = new Map(); for (const lowestEpoch of persistEpochs) { // usually there is only 0 or 1 epoch to persist in this loop - persistCount += await this.processPastEpoch(blockRootHex, state, lowestEpoch); + const prunedStates = await this.processPastEpoch(blockRootHex, state, lowestEpoch); + result.set(lowestEpoch, prunedStates); } - if (persistCount > 0) { - this.logger.verbose("Persisted checkpoint states", { - slot: blockSlot, - root: blockRootHex, - persistCount, - persistEpochs: persistEpochs.length, - }); - } - return persistCount; + return result; } /** @@ -648,13 +648,16 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * Performance note: * - In normal condition, we persist 1 checkpoint state per epoch. * - In reorged condition, we may persist multiple (most likely 2) checkpoint states per epoch. + * + * Return the pruned states from memory */ private async processPastEpoch( blockRootHex: RootHex, state: CachedBeaconStateAllForks, epoch: Epoch - ): Promise { + ): Promise { let persistCount = 0; + const prunedStates: CachedBeaconStateAllForks[] = []; const epochBoundarySlot = computeStartSlotAtEpoch(epoch); const epochBoundaryRoot = epochBoundarySlot === state.slot ? fromHexString(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot); @@ -735,10 +738,20 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.metrics?.statePruneFromMemoryCount.inc(); this.logger.verbose("Pruned checkpoint state from memory", logMeta); } + + prunedStates.push(state); } } - return persistCount; + if (persistCount > 0) { + this.logger.verbose("Persisted checkpoint states", { + stateSlot: state.slot, + blockRoot: blockRootHex, + persistCount, + }); + } + + return prunedStates; } /** diff --git a/packages/beacon-node/src/chain/stateCache/types.ts b/packages/beacon-node/src/chain/stateCache/types.ts index 1e8d6bd1bd62..cd93c34bde89 100644 --- a/packages/beacon-node/src/chain/stateCache/types.ts +++ b/packages/beacon-node/src/chain/stateCache/types.ts @@ -72,8 +72,11 @@ export interface CheckpointStateCache { ): Promise; updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null; prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void; - pruneFinalized(finalizedEpoch: Epoch): void; - processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise; + pruneFinalized(finalizedEpoch: Epoch): Map | null; + processState( + blockRootHex: RootHex, + state: CachedBeaconStateAllForks + ): Promise | null>; clear(): void; dumpSummary(): routes.lodestar.StateCacheItem[]; /** Expose beacon states stored in cache. Use with caution */ diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 737a900e5f64..6ade99eb7b6a 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1315,6 +1315,21 @@ export function createLodestarMetrics( }), }, + balancesTreeCache: { + size: register.gauge({ + name: "lodestar_balances_tree_cache_size", + help: "Balances tree cache size", + }), + hit: register.gauge({ + name: "lodestar_balances_tree_cache_hit_total", + help: "Total number of balances tree cache hits", + }), + miss: register.gauge({ + name: "lodestar_balances_tree_cache_miss_total", + help: "Total number of balances tree cache misses", + }), + }, + seenCache: { aggregatedAttestations: { superSetCheckTotal: register.histogram({ diff --git a/packages/state-transition/src/cache/balancesTreeCache.ts b/packages/state-transition/src/cache/balancesTreeCache.ts new file mode 100644 index 000000000000..0466824e490d --- /dev/null +++ b/packages/state-transition/src/cache/balancesTreeCache.ts @@ -0,0 +1,5 @@ +import {UintNumberType, ListBasicTreeViewDU} from "@chainsafe/ssz"; + +export interface IBalancesTreeCache { + getUnusedBalances(): ListBasicTreeViewDU | undefined; +} diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index af6e976e9089..4ad5be709bf9 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -60,6 +60,7 @@ import { SyncCommitteeCache, SyncCommitteeCacheEmpty, } from "./syncCommitteeCache.js"; +import {IBalancesTreeCache} from "./balancesTreeCache.js"; /** `= PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)` */ export const PROPOSER_WEIGHT_FACTOR = PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT); @@ -68,6 +69,7 @@ export type EpochCacheImmutableData = { config: BeaconConfig; pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; + balancesTreeCache?: IBalancesTreeCache; }; export type EpochCacheOpts = { @@ -129,6 +131,8 @@ export class EpochCache { */ unfinalizedPubkey2index: UnfinalizedPubkeyIndexMap; + balancesTreeCache?: IBalancesTreeCache; + /** * Indexes of the block proposers for the current epoch. * @@ -245,6 +249,7 @@ export class EpochCache { pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; unfinalizedPubkey2index: UnfinalizedPubkeyIndexMap; + balancesTreeCache?: IBalancesTreeCache; proposers: number[]; proposersPrevEpoch: number[] | null; proposersNextEpoch: ProposersDeferred; @@ -273,6 +278,7 @@ export class EpochCache { this.pubkey2index = data.pubkey2index; this.index2pubkey = data.index2pubkey; this.unfinalizedPubkey2index = data.unfinalizedPubkey2index; + this.balancesTreeCache = data.balancesTreeCache; this.proposers = data.proposers; this.proposersPrevEpoch = data.proposersPrevEpoch; this.proposersNextEpoch = data.proposersNextEpoch; @@ -306,7 +312,7 @@ export class EpochCache { */ static createFromState( state: BeaconStateAllForks, - {config, pubkey2index, index2pubkey}: EpochCacheImmutableData, + {config, pubkey2index, index2pubkey, balancesTreeCache}: EpochCacheImmutableData, opts?: EpochCacheOpts ): EpochCache { const currentEpoch = computeEpochAtSlot(state.slot); @@ -483,6 +489,7 @@ export class EpochCache { index2pubkey, // `createFromFinalizedState()` creates cache with empty unfinalizedPubkey2index. Be cautious to only pass in finalized state unfinalizedPubkey2index: newUnfinalizedPubkeyIndexMap(), + balancesTreeCache, proposers, // On first epoch, set to null to prevent unnecessary work since this is only used for metrics proposersPrevEpoch: null, @@ -524,6 +531,7 @@ export class EpochCache { index2pubkey: this.index2pubkey, // No need to clone this reference. On each mutation the `unfinalizedPubkey2index` reference is replaced, @see `addPubkey` unfinalizedPubkey2index: this.unfinalizedPubkey2index, + balancesTreeCache: this.balancesTreeCache, // Immutable data proposers: this.proposers, proposersPrevEpoch: this.proposersPrevEpoch, diff --git a/packages/state-transition/src/epoch/processRewardsAndPenalties.ts b/packages/state-transition/src/epoch/processRewardsAndPenalties.ts index ef074dfd6820..5b42f4175a04 100644 --- a/packages/state-transition/src/epoch/processRewardsAndPenalties.ts +++ b/packages/state-transition/src/epoch/processRewardsAndPenalties.ts @@ -39,7 +39,7 @@ export function processRewardsAndPenalties( // important: do not change state one balance at a time. Set them all at once, constructing the tree in one go // cache the balances array, too - state.balances = ssz.phase0.Balances.toViewDU(balances); + state.balances = ssz.phase0.Balances.toViewDU(balances, state.epochCtx.balancesTreeCache?.getUnusedBalances()); // For processEffectiveBalanceUpdates() to prevent having to re-compute the balances array. // For validator metrics diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index 4ed801e3c490..7fcc5a6c860a 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -42,6 +42,7 @@ export { EpochCacheErrorCode, } from "./cache/epochCache.js"; export {type EpochTransitionCache, beforeProcessEpoch} from "./cache/epochTransitionCache.js"; +export type {IBalancesTreeCache} from "./cache/balancesTreeCache.js"; // Aux data-structures export {