From 12e187148db713583e424c004277f51f4a634a85 Mon Sep 17 00:00:00 2001 From: twoeths Date: Fri, 2 Aug 2024 00:19:23 +0700 Subject: [PATCH] feat: improve before process epoch (#6979) * fix: do not populate proposerIndices and inclusionDelays from altair * feat: remove eligibleValidatorIndices * fix: avoid array.slice in processRegistryUpdates() * fix: reuse nextEpochShufflingActiveValidatorIndices * fix: state-transition check-types * chore: rename nextEpochShufflingActiveIndicesLength --- .../state-transition/src/cache/epochCache.ts | 13 ++++-- .../src/cache/epochTransitionCache.ts | 46 +++++++++---------- .../src/epoch/processInactivityUpdates.ts | 33 ++++++------- .../src/epoch/processRegistryUpdates.ts | 10 ++-- .../src/util/epochShuffling.ts | 12 +++-- .../test/perf/epoch/utilPhase0.ts | 9 +--- .../test/perf/util/shufflings.test.ts | 3 +- 7 files changed, 69 insertions(+), 57 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index ae2e5786ef28..8f15dab90535 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -338,11 +338,16 @@ export class EpochCache { throw Error("totalActiveBalanceIncrements >= Number.MAX_SAFE_INTEGER. MAX_EFFECTIVE_BALANCE is too low."); } - const currentShuffling = cachedCurrentShuffling ?? computeEpochShuffling(state, currentActiveIndices, currentEpoch); + const currentShuffling = + cachedCurrentShuffling ?? + computeEpochShuffling(state, currentActiveIndices, currentActiveIndices.length, currentEpoch); const previousShuffling = cachedPreviousShuffling ?? - (isGenesis ? currentShuffling : computeEpochShuffling(state, previousActiveIndices, previousEpoch)); - const nextShuffling = cachedNextShuffling ?? computeEpochShuffling(state, nextActiveIndices, nextEpoch); + (isGenesis + ? currentShuffling + : computeEpochShuffling(state, previousActiveIndices, previousActiveIndices.length, previousEpoch)); + const nextShuffling = + cachedNextShuffling ?? computeEpochShuffling(state, nextActiveIndices, nextActiveIndices.length, nextEpoch); const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER); @@ -501,6 +506,7 @@ export class EpochCache { state: BeaconStateAllForks, epochTransitionCache: { nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; + nextEpochShufflingActiveIndicesLength: number; nextEpochTotalActiveBalanceByIncrement: number; } ): void { @@ -512,6 +518,7 @@ export class EpochCache { this.nextShuffling = computeEpochShuffling( state, epochTransitionCache.nextEpochShufflingActiveValidatorIndices, + epochTransitionCache.nextEpochShufflingActiveIndicesLength, nextEpoch ); diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index dfe6bdd8e102..e6f84de6c62e 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -52,15 +52,6 @@ export interface EpochTransitionCache { }; currEpochUnslashedTargetStakeByIncrement: number; - /** - * Validator indices that are either - * - active in previous epoch - * - slashed and not yet withdrawable - * - * getRewardsAndPenalties() and processInactivityUpdates() iterate this list - */ - eligibleValidatorIndices: ValidatorIndex[]; - /** * Indices which will receive the slashing penalty * ``` @@ -125,10 +116,13 @@ export interface EpochTransitionCache { * - un-slashed validators * - prev attester flag set * With a status flag to check this conditions at once we just have to mask with an OR of the conditions. + * This is only for phase0 only. */ - proposerIndices: number[]; + /** + * This is for phase0 only. + */ inclusionDelays: number[]; flags: number[]; @@ -151,6 +145,11 @@ export interface EpochTransitionCache { */ nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; + /** + * We do not use up to `nextEpochShufflingActiveValidatorIndices.length`, use this to control that + */ + nextEpochShufflingActiveIndicesLength: number; + /** * Altair specific, this is total active balances for the next epoch. * This is only used in `afterProcessEpoch` to compute base reward and sync participant reward. @@ -191,12 +190,14 @@ const isActivePrevEpoch = new Array(); const isActiveCurrEpoch = new Array(); /** WARNING: reused, never gc'd */ const isActiveNextEpoch = new Array(); -/** WARNING: reused, never gc'd */ +/** WARNING: reused, never gc'd, from altair this is empty array */ const proposerIndices = new Array(); -/** WARNING: reused, never gc'd */ +/** WARNING: reused, never gc'd, from altair this is empty array */ const inclusionDelays = new Array(); /** WARNING: reused, never gc'd */ const flags = new Array(); +/** WARNING: reused, never gc'd */ +const nextEpochShufflingActiveValidatorIndices = new Array(); export function beforeProcessEpoch( state: CachedBeaconStateAllForks, @@ -212,12 +213,10 @@ export function beforeProcessEpoch( const slashingsEpoch = currentEpoch + intDiv(EPOCHS_PER_SLASHINGS_VECTOR, 2); - const eligibleValidatorIndices: ValidatorIndex[] = []; const indicesToSlash: ValidatorIndex[] = []; const indicesEligibleForActivationQueue: ValidatorIndex[] = []; const indicesEligibleForActivation: ValidatorIndex[] = []; const indicesToEject: ValidatorIndex[] = []; - const nextEpochShufflingActiveValidatorIndices: ValidatorIndex[] = []; let totalActiveStakeByIncrement = 0; @@ -227,6 +226,8 @@ export function beforeProcessEpoch( const validators = state.validators.getAllReadonlyValues(); const validatorCount = validators.length; + nextEpochShufflingActiveValidatorIndices.length = validatorCount; + let nextEpochShufflingActiveIndicesLength = 0; // pre-fill with true (most validators are active) isActivePrevEpoch.length = validatorCount; isActiveCurrEpoch.length = validatorCount; @@ -238,14 +239,10 @@ export function beforeProcessEpoch( // During the epoch transition, additional data is precomputed to avoid traversing any state a second // time. Attestations are a big part of this, and each validator has a "status" to represent its // precomputed participation. - // - proposerIndex: number; // -1 when not included by any proposer - // - inclusionDelay: number; + // - proposerIndex: number; // -1 when not included by any proposer, for phase0 only so it's declared inside phase0 block below + // - inclusionDelay: number;// for phase0 only so it's declared inside phase0 block below // - flags: number; // bitfield of AttesterFlags - proposerIndices.length = validatorCount; - inclusionDelays.length = validatorCount; flags.length = validatorCount; - proposerIndices.fill(-1); - inclusionDelays.fill(0); // flags.fill(0); // flags will be zero'd out below // In the first loop, set slashed+eligibility @@ -284,7 +281,6 @@ export function beforeProcessEpoch( // This is done to prevent self-slashing from being a way to escape inactivity leaks. // TODO: Consider using an array of `eligibleValidatorIndices: number[]` if (isActivePrev || (validator.slashed && prevEpoch + 1 < validator.withdrawableEpoch)) { - eligibleValidatorIndices.push(i); flag |= FLAG_ELIGIBLE_ATTESTER; } @@ -348,7 +344,7 @@ export function beforeProcessEpoch( } if (isActiveNext2) { - nextEpochShufflingActiveValidatorIndices.push(i); + nextEpochShufflingActiveValidatorIndices[nextEpochShufflingActiveIndicesLength++] = i; } } @@ -368,6 +364,10 @@ export function beforeProcessEpoch( ); if (forkSeq === ForkSeq.phase0) { + proposerIndices.length = validatorCount; + proposerIndices.fill(-1); + inclusionDelays.length = validatorCount; + inclusionDelays.fill(0); processPendingAttestations( state as CachedBeaconStatePhase0, proposerIndices, @@ -467,12 +467,12 @@ export function beforeProcessEpoch( headStakeByIncrement: prevHeadUnslStake, }, currEpochUnslashedTargetStakeByIncrement: currTargetUnslStake, - eligibleValidatorIndices, indicesToSlash, indicesEligibleForActivationQueue, indicesEligibleForActivation, indicesToEject, nextEpochShufflingActiveValidatorIndices, + nextEpochShufflingActiveIndicesLength, // to be updated in processEffectiveBalanceUpdates nextEpochTotalActiveBalanceByIncrement: 0, isActivePrevEpoch, diff --git a/packages/state-transition/src/epoch/processInactivityUpdates.ts b/packages/state-transition/src/epoch/processInactivityUpdates.ts index aedb077d6bbe..4a9b129ee793 100644 --- a/packages/state-transition/src/epoch/processInactivityUpdates.ts +++ b/packages/state-transition/src/epoch/processInactivityUpdates.ts @@ -24,30 +24,31 @@ export function processInactivityUpdates(state: CachedBeaconStateAltair, cache: const {config, inactivityScores} = state; const {INACTIVITY_SCORE_BIAS, INACTIVITY_SCORE_RECOVERY_RATE} = config; - const {flags, eligibleValidatorIndices} = cache; + const {flags} = cache; const inActivityLeak = isInInactivityLeak(state); // this avoids importing FLAG_ELIGIBLE_ATTESTER inside the for loop, check the compiled code - const {FLAG_PREV_TARGET_ATTESTER_UNSLASHED, hasMarkers} = attesterStatusUtil; + const {FLAG_PREV_TARGET_ATTESTER_UNSLASHED, FLAG_ELIGIBLE_ATTESTER, hasMarkers} = attesterStatusUtil; const inactivityScoresArr = inactivityScores.getAll(); - for (let j = 0; j < eligibleValidatorIndices.length; j++) { - const i = eligibleValidatorIndices[j]; + for (let i = 0; i < flags.length; i++) { const flag = flags[i]; - let inactivityScore = inactivityScoresArr[i]; + if (hasMarkers(flag, FLAG_ELIGIBLE_ATTESTER)) { + let inactivityScore = inactivityScoresArr[i]; - const prevInactivityScore = inactivityScore; - if (hasMarkers(flag, FLAG_PREV_TARGET_ATTESTER_UNSLASHED)) { - inactivityScore -= Math.min(1, inactivityScore); - } else { - inactivityScore += INACTIVITY_SCORE_BIAS; - } - if (!inActivityLeak) { - inactivityScore -= Math.min(INACTIVITY_SCORE_RECOVERY_RATE, inactivityScore); - } - if (inactivityScore !== prevInactivityScore) { - inactivityScores.set(i, inactivityScore); + const prevInactivityScore = inactivityScore; + if (hasMarkers(flag, FLAG_PREV_TARGET_ATTESTER_UNSLASHED)) { + inactivityScore -= Math.min(1, inactivityScore); + } else { + inactivityScore += INACTIVITY_SCORE_BIAS; + } + if (!inActivityLeak) { + inactivityScore -= Math.min(INACTIVITY_SCORE_RECOVERY_RATE, inactivityScore); + } + if (inactivityScore !== prevInactivityScore) { + inactivityScores.set(i, inactivityScore); + } } } } diff --git a/packages/state-transition/src/epoch/processRegistryUpdates.ts b/packages/state-transition/src/epoch/processRegistryUpdates.ts index 0591f982d1d5..905e7b567c01 100644 --- a/packages/state-transition/src/epoch/processRegistryUpdates.ts +++ b/packages/state-transition/src/epoch/processRegistryUpdates.ts @@ -37,9 +37,13 @@ export function processRegistryUpdates(state: CachedBeaconStateAllForks, cache: } const finalityEpoch = state.finalizedCheckpoint.epoch; + // this avoids an array allocation compared to `slice(0, epochCtx.activationChurnLimit)` + const len = Math.min(cache.indicesEligibleForActivation.length, epochCtx.activationChurnLimit); + const activationEpoch = computeActivationExitEpoch(cache.currentEpoch); // dequeue validators for activation up to churn limit - for (const index of cache.indicesEligibleForActivation.slice(0, epochCtx.activationChurnLimit)) { - const validator = validators.get(index); + for (let i = 0; i < len; i++) { + const validatorIndex = cache.indicesEligibleForActivation[i]; + const validator = validators.get(validatorIndex); // placement in queue is finalized if (validator.activationEligibilityEpoch > finalityEpoch) { // remaining validators all have an activationEligibilityEpoch that is higher anyway, break early @@ -48,6 +52,6 @@ export function processRegistryUpdates(state: CachedBeaconStateAllForks, cache: // So we need to filter by finalityEpoch here to comply with the spec. break; } - validator.activationEpoch = computeActivationExitEpoch(cache.currentEpoch); + validator.activationEpoch = activationEpoch; } } diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 12f270d29792..e101da38f297 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -62,16 +62,22 @@ export function computeCommitteeCount(activeValidatorCount: number): number { export function computeEpochShuffling( state: BeaconStateAllForks, activeIndices: ArrayLike, + activeValidatorCount: number, epoch: Epoch ): EpochShuffling { const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); - // copy - const _activeIndices = new Uint32Array(activeIndices); + if (activeValidatorCount > activeIndices.length) { + throw new Error(`Invalid activeValidatorCount: ${activeValidatorCount} > ${activeIndices.length}`); + } + // only the first `activeValidatorCount` elements are copied to `activeIndices` + const _activeIndices = new Uint32Array(activeValidatorCount); + for (let i = 0; i < activeValidatorCount; i++) { + _activeIndices[i] = activeIndices[i]; + } const shuffling = _activeIndices.slice(); unshuffleList(shuffling, seed); - const activeValidatorCount = activeIndices.length; const committeesPerSlot = computeCommitteeCount(activeValidatorCount); const committeeCount = committeesPerSlot * SLOTS_PER_EPOCH; diff --git a/packages/state-transition/test/perf/epoch/utilPhase0.ts b/packages/state-transition/test/perf/epoch/utilPhase0.ts index 026506510979..41c7d9780e01 100644 --- a/packages/state-transition/test/perf/epoch/utilPhase0.ts +++ b/packages/state-transition/test/perf/epoch/utilPhase0.ts @@ -1,4 +1,4 @@ -import {AttesterFlags, FLAG_ELIGIBLE_ATTESTER, hasMarkers, toAttesterFlags} from "../../../src/index.js"; +import {AttesterFlags, toAttesterFlags} from "../../../src/index.js"; import {CachedBeaconStatePhase0, CachedBeaconStateAltair, EpochTransitionCache} from "../../../src/types.js"; /** @@ -14,18 +14,11 @@ export function generateBalanceDeltasEpochTransitionCache( const vc = state.validators.length; const {proposerIndices, inclusionDelays, flags} = generateStatuses(state.validators.length, flagFactors); - const eligibleValidatorIndices: number[] = []; - for (let i = 0; i < flags.length; i++) { - if (hasMarkers(flags[i], FLAG_ELIGIBLE_ATTESTER)) { - eligibleValidatorIndices.push(i); - } - } const cache: Partial = { proposerIndices, inclusionDelays, flags, - eligibleValidatorIndices, totalActiveStakeByIncrement: vc, baseRewardPerIncrement: 726, prevEpochUnslashedStake: { diff --git a/packages/state-transition/test/perf/util/shufflings.test.ts b/packages/state-transition/test/perf/util/shufflings.test.ts index e04dd405d960..96c7878a46ac 100644 --- a/packages/state-transition/test/perf/util/shufflings.test.ts +++ b/packages/state-transition/test/perf/util/shufflings.test.ts @@ -35,7 +35,8 @@ describe("epoch shufflings", () => { itBench({ id: `computeEpochShuffling - vc ${numValidators}`, fn: () => { - computeEpochShuffling(state, state.epochCtx.nextShuffling.activeIndices, nextEpoch); + const {activeIndices} = state.epochCtx.nextShuffling; + computeEpochShuffling(state, activeIndices, activeIndices.length, nextEpoch); }, });