feat: add max activation churn limit and other deneb devnet-9 spec updates

packages/beacon-node/test/spec/presets/fork_choice.test.ts

@@ -4,8 +4,8 @@ import {toHexString} from "@chainsafe/ssz";
 import {BeaconStateAllForks, isExecutionStateType} from "@lodestar/state-transition";
 import {InputType} from "@lodestar/spec-test-util";
 import {CheckpointWithHex, ForkChoice} from "@lodestar/fork-choice";
-import {phase0, allForks, bellatrix, ssz, RootHex} from "@lodestar/types";
-import {bnToNum} from "@lodestar/utils";
+import {phase0, allForks, bellatrix, ssz, RootHex, deneb} from "@lodestar/types";
+import {bnToNum, fromHex} from "@lodestar/utils";
 import {createBeaconConfig} from "@lodestar/config";
 import {ACTIVE_PRESET, ForkSeq, isForkBlobs} from "@lodestar/params";
 import {BeaconChain} from "../../../src/chain/index.js";
@@ -34,6 +34,7 @@ import {specTestIterator} from "../utils/specTestIterator.js";
 const ANCHOR_STATE_FILE_NAME = "anchor_state";
 const ANCHOR_BLOCK_FILE_NAME = "anchor_block";
 const BLOCK_FILE_NAME = "^(block)_([0-9a-zA-Z]+)$";
+const BLOBS_FILE_NAME = "^(blobs)_([0-9a-zA-Z]+)$";
 const POW_BLOCK_FILE_NAME = "^(pow_block)_([0-9a-zA-Z]+)$";
 const ATTESTATION_FILE_NAME = "^(attestation)_([0-9a-zA-Z])+$";
 const ATTESTER_SLASHING_FILE_NAME = "^(attester_slashing)_([0-9a-zA-Z])+$";
@@ -147,6 +148,15 @@ const forkChoiceTest =
                 throw Error(`No block ${step.block}`);
               }
 
+              let blobs: deneb.Blob[] | undefined;
+              let proofs: deneb.KZGProof[] | undefined;
+              if (step.blobs !== undefined) {
+                blobs = testcase.blobs.get(step.blobs);
+              }
+              if (step.proofs !== undefined) {
+                proofs = step.proofs.map((proof) => ssz.deneb.KZGProof.deserialize(fromHex(proof)));
+              }
+
               const {slot} = signedBlock.message;
               // Log the BeaconBlock root instead of the SignedBeaconBlock root, forkchoice references BeaconBlock roots
               const blockRoot = config
@@ -160,19 +170,50 @@ const forkChoiceTest =
                 isValid,
               });
 
-              const blockImport =
-                config.getForkSeq(slot) < ForkSeq.deneb
-                  ? getBlockInput.preDeneb(config, signedBlock, BlockSource.gossip, null)
-                  : getBlockInput.postDeneb(
-                      config,
-                      signedBlock,
-                      BlockSource.gossip,
-                      ssz.deneb.BlobSidecars.defaultValue(),
-                      null,
-                      [null]
-                    );
-
               try {
+                let blockImport;
+                if (config.getForkSeq(slot) >= ForkSeq.deneb) {
+                  if (blobs === undefined) {
+                    // seems like some deneb tests don't have this and we are supposed to assume empty
+                    // throw Error("Missing blobs for the deneb+ block");
+                    blobs = [];
+                  }
+                  if (proofs === undefined) {
+                    // seems like some deneb tests don't have this and we are supposed to assume empty
+                    // throw Error("proofs for the deneb+ block");
+                    proofs = [];
+                  }
+                  // the kzg lib for validation of minimal setup is not yet integrated, lets just verify lengths
+                  // post integration use validateBlobsAndProofs
+                  const commitments = (signedBlock as deneb.SignedBeaconBlock).message.body.blobKzgCommitments;
+                  if (blobs.length !== commitments.length || proofs.length !== commitments.length) {
+                    throw Error("Invalid blobs or proofs lengths");
+                  }
+
+                  const blockRoot = config
+                    .getForkTypes(signedBlock.message.slot)
+                    .BeaconBlock.hashTreeRoot(signedBlock.message);
+                  const blobSidecars: deneb.BlobSidecars = blobs.map((blob, index) => {
+                    return {
+                      blockRoot,
+                      index,
+                      slot,
+                      blob,
+                      // proofs isn't undefined here but typescript(check types) can't figure it out
+                      kzgProof: (proofs ?? [])[index],
+                      kzgCommitment: commitments[index],
+                      blockParentRoot: signedBlock.message.parentRoot,
+                      proposerIndex: signedBlock.message.proposerIndex,
+                    };
+                  });
+
+                  blockImport = getBlockInput.postDeneb(config, signedBlock, BlockSource.gossip, blobSidecars, null, [
+                    null,
+                  ]);
+                } else {
+                  blockImport = getBlockInput.preDeneb(config, signedBlock, BlockSource.gossip, null);
+                }
+
                 await chain.processBlock(blockImport, {
                   seenTimestampSec: tickTime,
                   validBlobSidecars: true,
@@ -276,13 +317,15 @@ const forkChoiceTest =
           [ANCHOR_STATE_FILE_NAME]: ssz[fork].BeaconState,
           [ANCHOR_BLOCK_FILE_NAME]: ssz[fork].BeaconBlock,
           [BLOCK_FILE_NAME]: ssz[fork].SignedBeaconBlock,
+          [BLOBS_FILE_NAME]: ssz.deneb.Blobs,
           [POW_BLOCK_FILE_NAME]: ssz.bellatrix.PowBlock,
           [ATTESTATION_FILE_NAME]: ssz.phase0.Attestation,
           [ATTESTER_SLASHING_FILE_NAME]: ssz.phase0.AttesterSlashing,
         },
         mapToTestCase: (t: Record<string, any>) => {
           // t has input file name as key
           const blocks = new Map<string, allForks.SignedBeaconBlock>();
+          const blobs = new Map<string, deneb.Blobs>();
           const powBlocks = new Map<string, bellatrix.PowBlock>();
           const attestations = new Map<string, phase0.Attestation>();
           const attesterSlashings = new Map<string, phase0.AttesterSlashing>();
@@ -291,6 +334,10 @@ const forkChoiceTest =
             if (blockMatch) {
               blocks.set(key, t[key]);
             }
+            const blobsMatch = key.match(BLOBS_FILE_NAME);
+            if (blobsMatch) {
+              blobs.set(key, t[key]);
+            }
             const powBlockMatch = key.match(POW_BLOCK_FILE_NAME);
             if (powBlockMatch) {
               powBlocks.set(key, t[key]);
@@ -310,6 +357,7 @@ const forkChoiceTest =
             anchorBlock: t[ANCHOR_BLOCK_FILE_NAME] as ForkChoiceTestCase["anchorBlock"],
             steps: t["steps"] as ForkChoiceTestCase["steps"],
             blocks,
+            blobs,
             powBlocks,
             attestations,
             attesterSlashings,
@@ -319,6 +367,13 @@ const forkChoiceTest =
         // eslint-disable-next-line @typescript-eslint/no-empty-function
         expectFunc: () => {},
         // Do not manually skip tests here, do it in packages/beacon-node/test/spec/presets/index.test.ts
+        // EXCEPTION : this test skipped here because prefix match can't be don't for this particular test
+        // as testId for the entire directory is same : `deneb/fork_choice/on_block/pyspec_tests` and
+        // we just want to skip this one particular test because we don't have minimal kzg lib integrated
+        //
+        // This skip can be removed once c-kzg lib with run-time minimal blob size setup is released and
+        // integrated
+        shouldSkip: (_testcase, name, _index) => name.includes("invalid_incorrect_proof"),
       },
     };
   };
@@ -364,6 +419,8 @@ type OnAttesterSlashing = {
 type OnBlock = {
   /** the name of the `block_<32-byte-root>.ssz_snappy` file. To execute `on_block(store, block)` */
   block: string;
+  blobs?: string;
+  proofs?: string[];
   /** optional, default to `true`. */
   valid?: number;
 };
@@ -412,6 +469,7 @@ type ForkChoiceTestCase = {
   anchorBlock: allForks.BeaconBlock;
   steps: Step[];
   blocks: Map<string, allForks.SignedBeaconBlock>;
+  blobs: Map<string, deneb.Blobs>;
   powBlocks: Map<string, bellatrix.PowBlock>;
   attestations: Map<string, phase0.Attestation>;
   attesterSlashings: Map<string, phase0.AttesterSlashing>;

packages/beacon-node/test/spec/specTestVersioning.ts

@@ -15,7 +15,7 @@ import {DownloadTestsOptions} from "@lodestar/spec-test-util";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 export const ethereumConsensusSpecsTests: DownloadTestsOptions = {
-  specVersion: "v1.4.0-beta.1",
+  specVersion: "v1.4.0-beta.2-hotfix",
   // Target directory is the host package root: 'packages/*/spec-tests'
   outputDir: path.join(__dirname, "../../spec-tests"),
   specTestsRepoUrl: "https://github.com/ethereum/consensus-spec-tests",

packages/config/src/chainConfig/presets/mainnet.ts

@@ -68,6 +68,7 @@ export const chainConfig: ChainConfig = {
   EJECTION_BALANCE: 16000000000,
   // 2**2 (= 4)
   MIN_PER_EPOCH_CHURN_LIMIT: 4,
+  MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8,
   // 2**16 (= 65,536)
   CHURN_LIMIT_QUOTIENT: 65536,
   PROPOSER_SCORE_BOOST: 40,

packages/config/src/chainConfig/presets/minimal.ts

@@ -65,7 +65,8 @@ export const chainConfig: ChainConfig = {
   // 2**4 * 10**9 (= 16,000,000,000) Gwei
   EJECTION_BALANCE: 16000000000,
   // 2**2 (= 4)
-  MIN_PER_EPOCH_CHURN_LIMIT: 4,
+  MIN_PER_EPOCH_CHURN_LIMIT: 2,
+  MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 4,
   // [customized] scale queue churn at much lower validator counts for testing
   CHURN_LIMIT_QUOTIENT: 32,
   PROPOSER_SCORE_BOOST: 40,

packages/config/src/chainConfig/types.ts

@@ -53,6 +53,7 @@ export type ChainConfig = {
   INACTIVITY_SCORE_RECOVERY_RATE: number;
   EJECTION_BALANCE: number;
   MIN_PER_EPOCH_CHURN_LIMIT: number;
+  MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: number;
   CHURN_LIMIT_QUOTIENT: number;
 
   // Proposer boost
@@ -105,6 +106,7 @@ export const chainConfigTypes: SpecTypes<ChainConfig> = {
   INACTIVITY_SCORE_RECOVERY_RATE: "number",
   EJECTION_BALANCE: "number",
   MIN_PER_EPOCH_CHURN_LIMIT: "number",
+  MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: "number",
   CHURN_LIMIT_QUOTIENT: "number",
 
   // Proposer boost

packages/fork-choice/src/forkChoice/forkChoice.ts

@@ -357,7 +357,9 @@ export class ForkChoice implements IForkChoice {
     if (
       this.opts?.proposerBoostEnabled &&
       this.fcStore.currentSlot === slot &&
-      blockDelaySec < this.config.SECONDS_PER_SLOT / INTERVALS_PER_SLOT
+      blockDelaySec < this.config.SECONDS_PER_SLOT / INTERVALS_PER_SLOT &&
+      // only boost the first block we see
+      this.proposerBoostRoot === null
     ) {
       this.proposerBoostRoot = blockRootHex;
     }

packages/params/test/e2e/ensure-config-is-synced.test.ts

@@ -8,7 +8,7 @@ import {loadConfigYaml} from "../yaml.js";
 // Not e2e, but slow. Run with e2e tests
 
 /** https://github.com/ethereum/consensus-specs/releases */
-const specConfigCommit = "v1.4.0-beta.1";
+const specConfigCommit = "v1.4.0-beta.2";
 
 describe("Ensure config is synced", function () {
   this.timeout(60 * 1000);

packages/state-transition/src/cache/epochCache.ts

@@ -24,6 +24,7 @@ import {
   computeSyncPeriodAtEpoch,
   getSeed,
   computeProposers,
+  getActivationChurnLimit,
 } from "../util/index.js";
 import {computeEpochShuffling, EpochShuffling} from "../util/epochShuffling.js";
 import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../util/syncCommittee.js";
@@ -145,6 +146,11 @@ export class EpochCache {
    * change through the epoch. It's used in initiateValidatorExit(). Must be update after changing active indexes.
    */
   churnLimit: number;
+
+  /**
+   * Fork limited actual activationChurnLimit
+   */
+  activationChurnLimit: number;
   /**
    * Closest epoch with available churn for validators to exit at. May be updated every block as validators are
    * initiateValidatorExit(). This value may vary on each fork of the state.
@@ -204,6 +210,7 @@ export class EpochCache {
     baseRewardPerIncrement: number;
     totalActiveBalanceIncrements: number;
     churnLimit: number;
+    activationChurnLimit: number;
     exitQueueEpoch: Epoch;
     exitQueueChurn: number;
     currentTargetUnslashedBalanceIncrements: number;
@@ -228,6 +235,7 @@ export class EpochCache {
     this.baseRewardPerIncrement = data.baseRewardPerIncrement;
     this.totalActiveBalanceIncrements = data.totalActiveBalanceIncrements;
     this.churnLimit = data.churnLimit;
+    this.activationChurnLimit = data.activationChurnLimit;
     this.exitQueueEpoch = data.exitQueueEpoch;
     this.exitQueueChurn = data.exitQueueChurn;
     this.currentTargetUnslashedBalanceIncrements = data.currentTargetUnslashedBalanceIncrements;
@@ -364,6 +372,11 @@ export class EpochCache {
     // the first block of the epoch process_block() call. So churnLimit must be computed at the end of the before epoch
     // transition and the result is valid until the end of the next epoch transition
     const churnLimit = getChurnLimit(config, currentShuffling.activeIndices.length);
+    const activationChurnLimit = getActivationChurnLimit(
+      config,
+      config.getForkSeq(state.slot),
+      currentShuffling.activeIndices.length
+    );
     if (exitQueueChurn >= churnLimit) {
       exitQueueEpoch += 1;
       exitQueueChurn = 0;
@@ -405,6 +418,7 @@ export class EpochCache {
       baseRewardPerIncrement,
       totalActiveBalanceIncrements,
       churnLimit,
+      activationChurnLimit,
       exitQueueEpoch,
       exitQueueChurn,
       previousTargetUnslashedBalanceIncrements,
@@ -444,6 +458,7 @@ export class EpochCache {
       baseRewardPerIncrement: this.baseRewardPerIncrement,
       totalActiveBalanceIncrements: this.totalActiveBalanceIncrements,
       churnLimit: this.churnLimit,
+      activationChurnLimit: this.activationChurnLimit,
       exitQueueEpoch: this.exitQueueEpoch,
       exitQueueChurn: this.exitQueueChurn,
       previousTargetUnslashedBalanceIncrements: this.previousTargetUnslashedBalanceIncrements,
@@ -503,6 +518,11 @@ export class EpochCache {
     // the first block of the epoch process_block() call. So churnLimit must be computed at the end of the before epoch
     // transition and the result is valid until the end of the next epoch transition
     this.churnLimit = getChurnLimit(this.config, this.currentShuffling.activeIndices.length);
+    this.activationChurnLimit = getActivationChurnLimit(
+      this.config,
+      this.config.getForkSeq(state.slot),
+      this.currentShuffling.activeIndices.length
+    );
 
     // Maybe advance exitQueueEpoch at the end of the epoch if there haven't been any exists for a while
     const exitQueueEpoch = computeActivationExitEpoch(currEpoch);

packages/state-transition/src/epoch/processRegistryUpdates.ts

@@ -38,7 +38,7 @@ export function processRegistryUpdates(state: CachedBeaconStateAllForks, cache:
 
   const finalityEpoch = state.finalizedCheckpoint.epoch;
   // dequeue validators for activation up to churn limit
-  for (const index of cache.indicesEligibleForActivation.slice(0, epochCtx.churnLimit)) {
+  for (const index of cache.indicesEligibleForActivation.slice(0, epochCtx.activationChurnLimit)) {
     const validator = validators.get(index);
     // placement in queue is finalized
     if (validator.activationEligibilityEpoch > finalityEpoch) {

packages/state-transition/src/util/validator.ts

@@ -1,6 +1,7 @@
 import {Epoch, phase0, ValidatorIndex} from "@lodestar/types";
 import {intDiv} from "@lodestar/utils";
 import {ChainForkConfig} from "@lodestar/config";
+import {ForkSeq} from "@lodestar/params";
 import {BeaconStateAllForks} from "../types.js";
 
 /**
@@ -35,6 +36,14 @@ export function getActiveValidatorIndices(state: BeaconStateAllForks, epoch: Epo
   return indices;
 }
 
+export function getActivationChurnLimit(config: ChainForkConfig, fork: ForkSeq, activeValidatorCount: number): number {
+  if (fork >= ForkSeq.deneb) {
+    return Math.min(config.MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT, getChurnLimit(config, activeValidatorCount));
+  } else {
+    return getChurnLimit(config, activeValidatorCount);
+  }
+}
+
 export function getChurnLimit(config: ChainForkConfig, activeValidatorCount: number): number {
   return Math.max(config.MIN_PER_EPOCH_CHURN_LIMIT, intDiv(activeValidatorCount, config.CHURN_LIMIT_QUOTIENT));
 }

packages/validator/src/util/params.ts

@@ -118,6 +118,7 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record<keyof ConfigWit
     INACTIVITY_SCORE_RECOVERY_RATE: true,
     EJECTION_BALANCE: true,
     MIN_PER_EPOCH_CHURN_LIMIT: true,
+    MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: denebForkRelevant,
     CHURN_LIMIT_QUOTIENT: true,
 
     // Proposer boost