Init Fulu fork (#6677)

* fulu

* fix ci

* fix ncli

* attempt to fix ci

* fix wss_sim build

* fix vc api

* updated tests

* resolve merge conflicts

* fix tests

* fix check

* attempt to fix minimal test in CI

* addressed reviews

---------

Co-authored-by: tersec <tersec@users.noreply.github.com>

AllTests-mainnet.md

@@ -61,6 +61,10 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 + sanity check Electra blocks [Preset: mainnet]                                              OK
 + sanity check Electra states [Preset: mainnet]                                              OK
 + sanity check Electra states, reusing buffers [Preset: mainnet]                             OK
++ sanity check Fulu and cross-fork getState rollback [Preset: mainnet]                       OK
++ sanity check Fulu blocks [Preset: mainnet]                                                 OK
++ sanity check Fulu states [Preset: mainnet]                                                 OK
++ sanity check Fulu states, reusing buffers [Preset: mainnet]                                OK
 + sanity check blobs [Preset: mainnet]                                                       OK
 + sanity check genesis roundtrip [Preset: mainnet]                                           OK
 + sanity check phase 0 blocks [Preset: mainnet]                                              OK
@@ -69,7 +73,7 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 + sanity check phase 0 states, reusing buffers [Preset: mainnet]                             OK
 + sanity check state diff roundtrip [Preset: mainnet]                                        OK
 ```
-OK: 29/29 Fail: 0/29 Skip: 0/29
+OK: 33/33 Fail: 0/33 Skip: 0/33
 ## Beacon chain file test suite
 ```diff
 + Auto check/repair test (missing data)                                                      OK
@@ -107,8 +111,9 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 + Capella toSignedBlindedBeaconBlock                                                         OK
 + Deneb toSignedBlindedBeaconBlock                                                           OK
 + Electra toSignedBlindedBeaconBlock                                                         OK
++ Fulu toSignedBlindedBeaconBlock                                                            OK
 ```
-OK: 4/4 Fail: 0/4 Skip: 0/4
+OK: 5/5 Fail: 0/5 Skip: 0/5
 ## Block pool altair processing [Preset: mainnet]
 ```diff
 + Invalid signatures [Preset: mainnet]                                                       OK
@@ -1137,4 +1142,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 770/775 Fail: 0/775 Skip: 5/775
+OK: 775/780 Fail: 0/780 Skip: 5/780

beacon_chain/beacon_chain_db.nim

@@ -508,23 +508,43 @@ proc new*(T: type BeaconChainDB,
 
     # V1 - expected-to-be small rows get without rowid optimizations
     keyValues = kvStore db.openKvStore("key_values", true).expectDb()
-    blocks = [
+    blocks = if cfg.FULU_FORK_EPOCH != FAR_FUTURE_EPOCH: [
       kvStore db.openKvStore("blocks").expectDb(),
       kvStore db.openKvStore("altair_blocks").expectDb(),
       kvStore db.openKvStore("bellatrix_blocks").expectDb(),
       kvStore db.openKvStore("capella_blocks").expectDb(),
       kvStore db.openKvStore("deneb_blocks").expectDb(),
-      kvStore db.openKvStore("electra_blocks").expectDb()]
+      kvStore db.openKvStore("electra_blocks").expectDb(),
+      kvStore db.openKvStore("fulu_blocks").expectDb()]
+
+      else: [
+      kvStore db.openKvStore("blocks").expectDb(),
+      kvStore db.openKvStore("altair_blocks").expectDb(),
+      kvStore db.openKvStore("bellatrix_blocks").expectDb(),
+      kvStore db.openKvStore("capella_blocks").expectDb(),
+      kvStore db.openKvStore("deneb_blocks").expectDb(),
+      kvStore db.openKvStore("electra_blocks").expectDb(),
+      kvStore db.openKvStore("").expectDb()]
 
     stateRoots = kvStore db.openKvStore("state_roots", true).expectDb()
 
-    statesNoVal = [
-      kvStore db.openKvStore("state_no_validators2").expectDb(),
-      kvStore db.openKvStore("altair_state_no_validators").expectDb(),
-      kvStore db.openKvStore("bellatrix_state_no_validators").expectDb(),
-      kvStore db.openKvStore("capella_state_no_validator_pubkeys").expectDb(),
-      kvStore db.openKvStore("deneb_state_no_validator_pubkeys").expectDb(),
-      kvStore db.openKvStore("electra_state_no_validator_pubkeys").expectDb()]
+    statesNoVal = if cfg.FULU_FORK_EPOCH != FAR_FUTURE_EPOCH: [
+        kvStore db.openKvStore("state_no_validators").expectDb(),
+        kvStore db.openKvStore("altair_state_no_validators").expectDb(),
+        kvStore db.openKvStore("bellatrix_state_no_validators").expectDb(),
+        kvStore db.openKvStore("capella_state_no_validator_pubkeys").expectDb(),
+        kvStore db.openKvStore("deneb_state_no_validator_pubkeys").expectDb(),
+        kvStore db.openKvStore("electra_state_no_validator_pubkeys").expectDb(),
+        kvStore db.openKvStore("fulu_state_no_validator_pubkeys").expectDb()]
+
+      else: [
+        kvStore db.openKvStore("state_no_validators").expectDb(),
+        kvStore db.openKvStore("altair_state_no_validators").expectDb(),
+        kvStore db.openKvStore("bellatrix_state_no_validators").expectDb(),
+        kvStore db.openKvStore("capella_state_no_validator_pubkeys").expectDb(),
+        kvStore db.openKvStore("deneb_state_no_validator_pubkeys").expectDb(),
+        kvStore db.openKvStore("electra_state_no_validator_pubkeys").expectDb(),
+        kvStore db.openKvStore("").expectDb()]
 
     stateDiffs = kvStore db.openKvStore("state_diffs").expectDb()
     summaries = kvStore db.openKvStore("beacon_block_summaries", true).expectDb()
@@ -805,7 +825,7 @@ proc putBlock*(
     db: BeaconChainDB,
     value: bellatrix.TrustedSignedBeaconBlock |
            capella.TrustedSignedBeaconBlock | deneb.TrustedSignedBeaconBlock |
-           electra.TrustedSignedBeaconBlock) =
+           electra.TrustedSignedBeaconBlock | fulu.TrustedSignedBeaconBlock) =
   db.withManyWrites:
     db.blocks[type(value).kind].putSZSSZ(value.root.data, value)
     db.putBeaconBlockSummary(value.root, value.message.toBeaconBlockSummary())
@@ -859,6 +879,10 @@ template toBeaconStateNoImmutableValidators(state: electra.BeaconState):
     ElectraBeaconStateNoImmutableValidators =
   isomorphicCast[ElectraBeaconStateNoImmutableValidators](state)
 
+template toBeaconStateNoImmutableValidators(state: fulu.BeaconState):
+    FuluBeaconStateNoImmutableValidators =
+  isomorphicCast[FuluBeaconStateNoImmutableValidators](state)
+
 proc putState*(
     db: BeaconChainDB, key: Eth2Digest,
     value: phase0.BeaconState | altair.BeaconState) =
@@ -869,7 +893,7 @@ proc putState*(
 proc putState*(
     db: BeaconChainDB, key: Eth2Digest,
     value: bellatrix.BeaconState | capella.BeaconState | deneb.BeaconState |
-           electra.BeaconState) =
+           electra.BeaconState | fulu.BeaconState) =
   db.updateImmutableValidators(value.validators.asSeq())
   db.statesNoVal[type(value).kind].putSZSSZ(
     key.data, toBeaconStateNoImmutableValidators(value))
@@ -998,7 +1022,8 @@ proc getBlock*(
 
 proc getBlock*[
     X: bellatrix.TrustedSignedBeaconBlock | capella.TrustedSignedBeaconBlock |
-       deneb.TrustedSignedBeaconBlock | electra.TrustedSignedBeaconBlock](
+       deneb.TrustedSignedBeaconBlock | electra.TrustedSignedBeaconBlock |
+       fulu.TrustedSignedBeaconBlock](
     db: BeaconChainDB, key: Eth2Digest,
     T: type X): Opt[T] =
   # We only store blocks that we trust in the database
@@ -1053,7 +1078,8 @@ proc getBlockSSZ*(
 
 proc getBlockSSZ*[
     X: bellatrix.TrustedSignedBeaconBlock | capella.TrustedSignedBeaconBlock |
-       deneb.TrustedSignedBeaconBlock | electra.TrustedSignedBeaconBlock](
+       deneb.TrustedSignedBeaconBlock | electra.TrustedSignedBeaconBlock |
+       fulu.TrustedSignedBeaconBlock](
     db: BeaconChainDB, key: Eth2Digest, data: var seq[byte], T: type X): bool =
   let dataPtr = addr data # Short-lived
   var success = true
@@ -1102,7 +1128,8 @@ proc getBlockSZ*(
 
 proc getBlockSZ*[
     X: bellatrix.TrustedSignedBeaconBlock | capella.TrustedSignedBeaconBlock |
-       deneb.TrustedSignedBeaconBlock | electra.TrustedSignedBeaconBlock](
+       deneb.TrustedSignedBeaconBlock | electra.TrustedSignedBeaconBlock |
+       fulu.TrustedSignedBeaconBlock](
     db: BeaconChainDB, key: Eth2Digest, data: var seq[byte], T: type X): bool =
   let dataPtr = addr data # Short-lived
   func decode(data: openArray[byte]) =
@@ -1200,7 +1227,8 @@ proc getStateOnlyMutableValidators(
 proc getStateOnlyMutableValidators(
     immutableValidators: openArray[ImmutableValidatorData2],
     store: KvStoreRef, key: openArray[byte],
-    output: var (capella.BeaconState | deneb.BeaconState | electra.BeaconState),
+    output: var (capella.BeaconState | deneb.BeaconState | electra.BeaconState |
+                 fulu.BeaconState),
     rollback: RollbackProc): bool =
   ## Load state into `output` - BeaconState is large so we want to avoid
   ## re-allocating it if possible
@@ -1285,7 +1313,8 @@ proc getState*(
 proc getState*(
     db: BeaconChainDB, key: Eth2Digest,
     output: var (altair.BeaconState | bellatrix.BeaconState |
-                 capella.BeaconState | deneb.BeaconState | electra.BeaconState),
+                 capella.BeaconState | deneb.BeaconState | electra.BeaconState |
+                 fulu.BeaconState),
     rollback: RollbackProc): bool =
   ## Load state into `output` - BeaconState is large so we want to avoid
   ## re-allocating it if possible
@@ -1365,7 +1394,7 @@ proc containsBlock*(
 proc containsBlock*[
     X: altair.TrustedSignedBeaconBlock | bellatrix.TrustedSignedBeaconBlock |
        capella.TrustedSignedBeaconBlock | deneb.TrustedSignedBeaconBlock |
-       electra.TrustedSignedBeaconBlock](
+       electra.TrustedSignedBeaconBlock | fulu.TrustedSignedBeaconBlock](
     db: BeaconChainDB, key: Eth2Digest, T: type X): bool =
   db.blocks[X.kind].contains(key.data).expectDb()
 
@@ -1506,7 +1535,7 @@ iterator getAncestorSummaries*(db: BeaconChainDB, root: Eth2Digest):
 
   # Backwards compat for reading old databases, or those that for whatever
   # reason lost a summary along the way..
-  static: doAssert ConsensusFork.high == ConsensusFork.Electra
+  static: doAssert ConsensusFork.high == ConsensusFork.Fulu
   while true:
     if db.v0.backend.getSnappySSZ(
         subkey(BeaconBlockSummary, res.root), res.summary) == GetResult.found:
@@ -1523,6 +1552,8 @@ iterator getAncestorSummaries*(db: BeaconChainDB, root: Eth2Digest):
       res.summary = blck.get().message.toBeaconBlockSummary()
     elif (let blck = db.getBlock(res.root, electra.TrustedSignedBeaconBlock); blck.isSome()):
       res.summary = blck.get().message.toBeaconBlockSummary()
+    elif (let blck = db.getBlock(res.root, fulu.TrustedSignedBeaconBlock); blck.isSome()):
+      res.summary = blck.get().message.toBeaconBlockSummary()
     else:
       break
 

beacon_chain/beacon_chain_db_immutable.nim

@@ -17,6 +17,8 @@ from ./spec/datatypes/deneb import ExecutionPayloadHeader
 from ./spec/datatypes/electra import
   ExecutionPayloadHeader, PendingConsolidation, PendingDeposit,
   PendingPartialWithdrawal
+from ./spec/datatypes/fulu import
+  ExecutionPayloadHeader
 
 type
   # https://github.com/ethereum/consensus-specs/blob/v1.4.0/specs/phase0/beacon-chain.md#beaconstate
@@ -416,3 +418,87 @@ type
     pending_consolidations*:
       HashList[PendingConsolidation, Limit PENDING_CONSOLIDATIONS_LIMIT]
       ## [New in Electra:EIP7251]
+
+  # Memory-representation-equivalent to a Fulu BeaconState for in-place SSZ
+  # reading and writing
+  FuluBeaconStateNoImmutableValidators* = object
+    # Versioning
+    genesis_time*: uint64
+    genesis_validators_root*: Eth2Digest
+    slot*: Slot
+    fork*: Fork
+
+    # History
+    latest_block_header*: BeaconBlockHeader
+      ## `latest_block_header.state_root == ZERO_HASH` temporarily
+
+    block_roots*: HashArray[Limit SLOTS_PER_HISTORICAL_ROOT, Eth2Digest]
+      ## Needed to process attestations, older to newer
+
+    state_roots*: HashArray[Limit SLOTS_PER_HISTORICAL_ROOT, Eth2Digest]
+    historical_roots*: HashList[Eth2Digest, Limit HISTORICAL_ROOTS_LIMIT]
+      ## Frozen in Capella, replaced by historical_summaries
+
+    # Eth1
+    eth1_data*: Eth1Data
+    eth1_data_votes*:
+      HashList[Eth1Data, Limit(EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)]
+    eth1_deposit_index*: uint64
+
+    # Registry
+    validators*:
+      HashList[ValidatorStatusCapella, Limit VALIDATOR_REGISTRY_LIMIT]
+    balances*: HashList[Gwei, Limit VALIDATOR_REGISTRY_LIMIT]
+
+    # Randomness
+    randao_mixes*: HashArray[Limit EPOCHS_PER_HISTORICAL_VECTOR, Eth2Digest]
+
+    # Slashings
+    slashings*: HashArray[Limit EPOCHS_PER_SLASHINGS_VECTOR, Gwei]
+      ## Per-epoch sums of slashed effective balances
+
+    # Participation
+    previous_epoch_participation*: EpochParticipationFlags
+    current_epoch_participation*: EpochParticipationFlags
+
+    # Finality
+    justification_bits*: JustificationBits
+      ## Bit set for every recent justified epoch
+
+    previous_justified_checkpoint*: Checkpoint
+    current_justified_checkpoint*: Checkpoint
+    finalized_checkpoint*: Checkpoint
+
+    # Inactivity
+    inactivity_scores*: HashList[uint64, Limit VALIDATOR_REGISTRY_LIMIT]
+
+    # Light client sync committees
+    current_sync_committee*: SyncCommittee
+    next_sync_committee*: SyncCommittee
+
+    # Execution
+    latest_execution_payload_header*: fulu.ExecutionPayloadHeader
+
+    # Withdrawals
+    next_withdrawal_index*: WithdrawalIndex
+    next_withdrawal_validator_index*: uint64
+
+    # Deep history valid from Capella onwards
+    historical_summaries*:
+      HashList[HistoricalSummary, Limit HISTORICAL_ROOTS_LIMIT]
+
+    deposit_requests_start_index*: uint64  # [New in Electra:EIP6110]
+    deposit_balance_to_consume*: Gwei  # [New in Electra:EIP7251]
+    exit_balance_to_consume*: Gwei  # [New in Electra:EIP7251]
+    earliest_exit_epoch*: Epoch  # [New in Electra:EIP7251]
+    consolidation_balance_to_consume*: Gwei  # [New in Electra:EIP7251]
+    earliest_consolidation_epoch*: Epoch  # [New in Electra:EIP7251]
+    pending_deposits*: HashList[PendingDeposit, Limit PENDING_DEPOSITS_LIMIT]
+      ## [New in Electra:EIP7251]
+
+    # [New in Electra:EIP7251]
+    pending_partial_withdrawals*:
+      HashList[PendingPartialWithdrawal, Limit PENDING_PARTIAL_WITHDRAWALS_LIMIT]
+    pending_consolidations*:
+      HashList[PendingConsolidation, Limit PENDING_CONSOLIDATIONS_LIMIT]
+      ## [New in Electra:EIP7251]

beacon_chain/beacon_chain_file.nim

@@ -89,6 +89,8 @@ func getBlobForkCode(fork: ConsensusFork): uint64 =
     uint64(MaxForksCount)
   of ConsensusFork.Electra:
     uint64(MaxForksCount) + uint64(fork) - uint64(ConsensusFork.Deneb)
+  of ConsensusFork.Fulu:
+    uint64(MaxForksCount) + uint64(fork) - uint64(ConsensusFork.Electra)
   of ConsensusFork.Phase0 .. ConsensusFork.Capella:
     raiseAssert "Blobs are not supported for the fork"
 

beacon_chain/consensus_object_pools/attestation_pool.nim

@@ -652,7 +652,7 @@ func init(
     T: type AttestationCache,
     state: altair.HashedBeaconState | bellatrix.HashedBeaconState |
            capella.HashedBeaconState | deneb.HashedBeaconState |
-           electra.HashedBeaconState,
+           electra.HashedBeaconState | fulu.HashedBeaconState,
     cache: var StateCache): T =
   # Load attestations that are scheduled for being given rewards for
   let
@@ -861,7 +861,8 @@ proc getAttestationsForBlock*(pool: var AttestationPool,
       default(seq[phase0.Attestation])
 
 proc getElectraAttestationsForBlock*(
-    pool: var AttestationPool, state: electra.HashedBeaconState,
+    pool: var AttestationPool,
+    state: electra.HashedBeaconState | fulu.HashedBeaconState,
     cache: var StateCache): seq[electra.Attestation] =
   let newBlockSlot = state.data.slot.uint64
 

beacon_chain/consensus_object_pools/blob_quarantine.nim

@@ -72,7 +72,8 @@ func hasBlob*(
 
 func popBlobs*(
     quarantine: var BlobQuarantine, digest: Eth2Digest,
-    blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock):
+    blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock |
+          fulu.SignedBeaconBlock):
     seq[ref BlobSidecar] =
   var r: seq[ref BlobSidecar] = @[]
   for idx, kzg_commitment in blck.message.body.blob_kzg_commitments:
@@ -82,14 +83,18 @@ func popBlobs*(
   r
 
 func hasBlobs*(quarantine: BlobQuarantine,
-    blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock): bool =
+    blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock |
+          fulu.SignedBeaconBlock): bool =
+    # Having a fulu SignedBeaconBlock is incorrect atm, but
+    # shall be fixed once data columns are rebased to fulu
   for idx, kzg_commitment in blck.message.body.blob_kzg_commitments:
     if (blck.root, BlobIndex idx, kzg_commitment) notin quarantine.blobs:
       return false
   true
 
 func blobFetchRecord*(quarantine: BlobQuarantine,
-    blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock): BlobFetchRecord =
+    blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock |
+          fulu.SignedBeaconBlock): BlobFetchRecord =
   var indices: seq[BlobIndex]
   for i in 0..<len(blck.message.body.blob_kzg_commitments):
     let idx = BlobIndex(i)

beacon_chain/consensus_object_pools/block_clearance.nim

@@ -455,7 +455,9 @@ proc addBackfillBlock*(
   ok()
 
 template BlockAdded(kind: static ConsensusFork): untyped =
-  when kind == ConsensusFork.Electra:
+  when kind == ConsensusFork.Fulu:
+    OnFuluBlockAdded
+  elif kind == ConsensusFork.Electra:
     OnElectraBlockAdded
   elif kind == ConsensusFork.Deneb:
     OnDenebBlockAdded

beacon_chain/consensus_object_pools/block_dag.nim

@@ -70,7 +70,8 @@ func init*(
     blck: bellatrix.SomeBeaconBlock | bellatrix.TrustedBeaconBlock |
           capella.SomeBeaconBlock | capella.TrustedBeaconBlock |
           deneb.SomeBeaconBlock | deneb.TrustedBeaconBlock |
-          electra.SomeBeaconBlock | electra.TrustedBeaconBlock): BlockRef =
+          electra.SomeBeaconBlock | electra.TrustedBeaconBlock |
+          fulu.SomeBeaconBlock | fulu.TrustedBeaconBlock): BlockRef =
   BlockRef.init(
     root, Opt.some blck.body.execution_payload.block_hash,
     executionValid =

beacon_chain/consensus_object_pools/block_pools_types.nim

@@ -298,10 +298,12 @@ type
   OnCapellaBlockAdded* = OnBlockAdded[capella.TrustedSignedBeaconBlock]
   OnDenebBlockAdded* = OnBlockAdded[deneb.TrustedSignedBeaconBlock]
   OnElectraBlockAdded* = OnBlockAdded[electra.TrustedSignedBeaconBlock]
+  OnFuluBlockAdded* = OnBlockAdded[fulu.TrustedSignedBeaconBlock]
 
   OnForkyBlockAdded* =
     OnPhase0BlockAdded | OnAltairBlockAdded | OnBellatrixBlockAdded |
-    OnCapellaBlockAdded | OnDenebBlockAdded | OnElectraBlockAdded
+    OnCapellaBlockAdded | OnDenebBlockAdded | OnElectraBlockAdded |
+    OnFuluBlockAdded
 
   OnForkedBlockAdded* = proc(
     blckRef: BlockRef, blck: ForkedTrustedSignedBeaconBlock, epochRef: EpochRef,
@@ -340,7 +342,9 @@ type
     optimistic* {.serializedFieldName: "execution_optimistic".}: Option[bool]
 
 template OnBlockAddedCallback*(kind: static ConsensusFork): auto =
-  when kind == ConsensusFork.Electra:
+  when kind == ConsensusFork.Fulu:
+    typedesc[OnFuluBlockAdded]
+  elif kind == ConsensusFork.Electra:
     typedesc[OnElectraBlockAdded]
   elif kind == ConsensusFork.Deneb:
     typedesc[OnDenebBlockAdded]