Electra other containers (sigp#5652)

* add new fields to execution payload and header

* beacon state changes

* partial beacon state

* safe arith in upgrade to electra

* initialize balances cache in interop genesis state

* Revert "initialize balances cache in interop genesis state"

This reverts commit c60b522.

* always initialize balances cache if necessary in electra upgrade

* build cache earlier

* fix block test

* per fork NUM_FIELDS_POW2

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into electra-other-containers

* fix lints

* get fields based on post state, as is spec'd

* fix type and move cache build

beacon_node/execution_layer/src/engine_api.rs

@@ -632,6 +632,9 @@ impl<E: EthSpec> ExecutionPayloadBodyV1<E> {
                         withdrawals,
                         blob_gas_used: header.blob_gas_used,
                         excess_blob_gas: header.excess_blob_gas,
+                        // TODO(electra)
+                        deposit_receipts: <_>::default(),
+                        withdrawal_requests: <_>::default(),
                     }))
                 } else {
                     Err(format!(

beacon_node/execution_layer/src/engine_api/json_structures.rs

@@ -319,6 +319,9 @@ impl<E: EthSpec> From<JsonExecutionPayloadV4<E>> for ExecutionPayloadElectra<E>
                 .into(),
             blob_gas_used: payload.blob_gas_used,
             excess_blob_gas: payload.excess_blob_gas,
+            // TODO(electra)
+            deposit_receipts: Default::default(),
+            withdrawal_requests: Default::default(),
         }
     }
 }

beacon_node/execution_layer/src/lib.rs

@@ -2003,6 +2003,11 @@ impl<E: EthSpec> ExecutionLayer<E> {
                     withdrawals,
                     blob_gas_used: electra_block.blob_gas_used,
                     excess_blob_gas: electra_block.excess_blob_gas,
+                    // TODO(electra)
+                    // deposit_receipts: electra_block.deposit_receipts,
+                    // withdrawal_requests: electra_block.withdrawal_requests,
+                    deposit_receipts: <_>::default(),
+                    withdrawal_requests: <_>::default(),
                 })
             }
         };

beacon_node/execution_layer/src/test_utils/execution_block_generator.rs

@@ -659,6 +659,8 @@ impl<E: EthSpec> ExecutionBlockGenerator<E> {
                     withdrawals: pa.withdrawals.clone().into(),
                     blob_gas_used: 0,
                     excess_blob_gas: 0,
+                    deposit_receipts: vec![].into(),
+                    withdrawal_requests: vec![].into(),
                 }),
                 _ => unreachable!(),
             },

beacon_node/store/src/partial_beacon_state.rs

@@ -118,6 +118,29 @@ where
     #[ssz(skip_serializing, skip_deserializing)]
     #[superstruct(only(Capella, Deneb, Electra))]
     pub historical_summaries: Option<List<HistoricalSummary, E::HistoricalRootsLimit>>,
+
+    // Electra
+    #[superstruct(only(Electra))]
+    pub deposit_receipts_start_index: u64,
+    #[superstruct(only(Electra))]
+    pub deposit_balance_to_consume: u64,
+    #[superstruct(only(Electra))]
+    pub exit_balance_to_consume: u64,
+    #[superstruct(only(Electra))]
+    pub earliest_exit_epoch: Epoch,
+    #[superstruct(only(Electra))]
+    pub consolidation_balance_to_consume: u64,
+    #[superstruct(only(Electra))]
+    pub earliest_consolidation_epoch: Epoch,
+
+    // TODO(electra) should these be optional?
+    #[superstruct(only(Electra))]
+    pub pending_balance_deposits: List<PendingBalanceDeposit, E::PendingBalanceDepositsLimit>,
+    #[superstruct(only(Electra))]
+    pub pending_partial_withdrawals:
+        List<PendingPartialWithdrawal, E::PendingPartialWithdrawalsLimit>,
+    #[superstruct(only(Electra))]
+    pub pending_consolidations: List<PendingConsolidation, E::PendingConsolidationsLimit>,
 }
 
 /// Implement the conversion function from BeaconState -> PartialBeaconState.
@@ -261,7 +284,16 @@ impl<E: EthSpec> PartialBeaconState<E> {
                     inactivity_scores,
                     latest_execution_payload_header,
                     next_withdrawal_index,
-                    next_withdrawal_validator_index
+                    next_withdrawal_validator_index,
+                    deposit_receipts_start_index,
+                    deposit_balance_to_consume,
+                    exit_balance_to_consume,
+                    earliest_exit_epoch,
+                    consolidation_balance_to_consume,
+                    earliest_consolidation_epoch,
+                    pending_balance_deposits,
+                    pending_partial_withdrawals,
+                    pending_consolidations
                 ],
                 [historical_summaries]
             ),
@@ -525,7 +557,16 @@ impl<E: EthSpec> TryInto<BeaconState<E>> for PartialBeaconState<E> {
                     inactivity_scores,
                     latest_execution_payload_header,
                     next_withdrawal_index,
-                    next_withdrawal_validator_index
+                    next_withdrawal_validator_index,
+                    deposit_receipts_start_index,
+                    deposit_balance_to_consume,
+                    exit_balance_to_consume,
+                    earliest_exit_epoch,
+                    consolidation_balance_to_consume,
+                    earliest_consolidation_epoch,
+                    pending_balance_deposits,
+                    pending_partial_withdrawals,
+                    pending_consolidations
                 ],
                 [historical_summaries]
             ),

consensus/state_processing/src/upgrade/electra.rs

@@ -1,3 +1,4 @@
+use safe_arith::SafeArith;
 use std::mem;
 use types::{
     BeaconState, BeaconStateElectra, BeaconStateError as Error, ChainSpec, EpochCache, EthSpec,
@@ -10,14 +11,28 @@ pub fn upgrade_to_electra<E: EthSpec>(
     spec: &ChainSpec,
 ) -> Result<(), Error> {
     let epoch = pre_state.current_epoch();
-    let pre = pre_state.as_deneb_mut()?;
 
+    let earliest_exit_epoch = pre_state
+        .validators()
+        .iter()
+        .filter(|v| v.exit_epoch != spec.far_future_epoch)
+        .map(|v| v.exit_epoch)
+        .max()
+        .unwrap_or(epoch)
+        .safe_add(1)?;
+
+    // The total active balance cache must be built before the consolidation churn limit
+    // is calculated.
+    pre_state.build_total_active_balance_cache(spec)?;
+    let earliest_consolidation_epoch = spec.compute_activation_exit_epoch(epoch)?;
+
+    let pre = pre_state.as_deneb_mut()?;
     // Where possible, use something like `mem::take` to move fields from behind the &mut
     // reference. For other fields that don't have a good default value, use `clone`.
     //
     // Fixed size vectors get cloned because replacing them would require the same size
     // allocation as cloning.
-    let post = BeaconState::Electra(BeaconStateElectra {
+    let mut post = BeaconState::Electra(BeaconStateElectra {
         // Versioning
         genesis_time: pre.genesis_time,
         genesis_validators_root: pre.genesis_validators_root,
@@ -62,6 +77,16 @@ pub fn upgrade_to_electra<E: EthSpec>(
         next_withdrawal_index: pre.next_withdrawal_index,
         next_withdrawal_validator_index: pre.next_withdrawal_validator_index,
         historical_summaries: pre.historical_summaries.clone(),
+        // Electra
+        deposit_receipts_start_index: spec.unset_deposit_receipts_start_index,
+        deposit_balance_to_consume: 0,
+        exit_balance_to_consume: 0,
+        earliest_exit_epoch,
+        consolidation_balance_to_consume: 0,
+        earliest_consolidation_epoch,
+        pending_balance_deposits: Default::default(),
+        pending_partial_withdrawals: Default::default(),
+        pending_consolidations: Default::default(),
         // Caches
         total_active_balance: pre.total_active_balance,
         progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache),
@@ -71,6 +96,39 @@ pub fn upgrade_to_electra<E: EthSpec>(
         slashings_cache: mem::take(&mut pre.slashings_cache),
         epoch_cache: EpochCache::default(),
     });
+    *post.exit_balance_to_consume_mut()? = post.get_activation_exit_churn_limit(spec)?;
+    *post.consolidation_balance_to_consume_mut()? = post.get_consolidation_churn_limit(spec)?;
+
+    // Add validators that are not yet active to pending balance deposits
+    let validators = post.validators().clone();
+    let mut pre_activation = validators
+        .iter()
+        .enumerate()
+        .filter(|(_, validator)| validator.activation_epoch == spec.far_future_epoch)
+        .collect::<Vec<_>>();
+
+    // Sort the indices by activation_eligibility_epoch and then by index
+    pre_activation.sort_by(|(index_a, val_a), (index_b, val_b)| {
+        if val_a.activation_eligibility_epoch == val_b.activation_eligibility_epoch {
+            index_a.cmp(index_b)
+        } else {
+            val_a
+                .activation_eligibility_epoch
+                .cmp(&val_b.activation_eligibility_epoch)
+        }
+    });
+
+    // Process validators to queue entire balance and reset them
+    for (index, _) in pre_activation {
+        post.queue_entire_balance_and_reset_validator(index, spec)?;
+    }
+
+    // Ensure early adopters of compounding credentials go through the activation churn
+    for (index, validator) in validators.iter().enumerate() {
+        if validator.has_compounding_withdrawal_credential(spec) {
+            post.queue_excess_active_balance(index, spec)?;
+        }
+    }
 
     *pre_state = post;
 

consensus/types/src/beacon_block.rs

@@ -1074,9 +1074,8 @@ mod tests {
                     .expect("good electra block can be decoded"),
                 good_block
             );
-            // TODO(electra): once the Electra block is changed from Deneb, update this to match
-            // the other forks.
-            assert!(BeaconBlock::from_ssz_bytes(&bad_block.as_ssz_bytes(), &spec).is_ok());
+            BeaconBlock::from_ssz_bytes(&bad_block.as_ssz_bytes(), &spec)
+                .expect_err("bad electra block cannot be decoded");
         }
     }
 }

consensus/types/src/beacon_state.rs

@@ -467,6 +467,40 @@ where
     #[test_random(default)]
     pub historical_summaries: List<HistoricalSummary, E::HistoricalRootsLimit>,
 
+    // Electra
+    #[superstruct(only(Electra), partial_getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub deposit_receipts_start_index: u64,
+    #[superstruct(only(Electra), partial_getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub deposit_balance_to_consume: u64,
+    #[superstruct(only(Electra), partial_getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub exit_balance_to_consume: u64,
+    #[superstruct(only(Electra), partial_getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
+    pub earliest_exit_epoch: Epoch,
+    #[superstruct(only(Electra), partial_getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub consolidation_balance_to_consume: u64,
+    #[superstruct(only(Electra), partial_getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
+    pub earliest_consolidation_epoch: Epoch,
+    #[test_random(default)]
+    #[superstruct(only(Electra))]
+    pub pending_balance_deposits: List<PendingBalanceDeposit, E::PendingBalanceDepositsLimit>,
+    #[test_random(default)]
+    #[superstruct(only(Electra))]
+    pub pending_partial_withdrawals:
+        List<PendingPartialWithdrawal, E::PendingPartialWithdrawalsLimit>,
+    #[test_random(default)]
+    #[superstruct(only(Electra))]
+    pub pending_consolidations: List<PendingConsolidation, E::PendingConsolidationsLimit>,
+
     // Caching (not in the spec)
     #[serde(skip_serializing, skip_deserializing)]
     #[ssz(skip_serializing, skip_deserializing)]
@@ -2031,6 +2065,83 @@ impl<E: EthSpec> BeaconState<E> {
         self.epoch_cache().get_base_reward(validator_index)
     }
 
+    // ******* Electra accessors *******
+
+    /// Return the churn limit for the current epoch.
+    pub fn get_balance_churn_limit(&self, spec: &ChainSpec) -> Result<u64, Error> {
+        let total_active_balance = self.get_total_active_balance()?;
+        let churn = std::cmp::max(
+            spec.min_per_epoch_churn_limit_electra,
+            total_active_balance.safe_div(spec.churn_limit_quotient)?,
+        );
+
+        Ok(churn.safe_sub(churn.safe_rem(spec.effective_balance_increment)?)?)
+    }
+
+    /// Return the churn limit for the current epoch dedicated to activations and exits.
+    pub fn get_activation_exit_churn_limit(&self, spec: &ChainSpec) -> Result<u64, Error> {
+        Ok(std::cmp::min(
+            spec.max_per_epoch_activation_exit_churn_limit,
+            self.get_balance_churn_limit(spec)?,
+        ))
+    }
+
+    pub fn get_consolidation_churn_limit(&self, spec: &ChainSpec) -> Result<u64, Error> {
+        self.get_balance_churn_limit(spec)?
+            .safe_sub(self.get_activation_exit_churn_limit(spec)?)
+            .map_err(Into::into)
+    }
+
+    // ******* Electra mutators *******
+
+    pub fn queue_excess_active_balance(
+        &mut self,
+        validator_index: usize,
+        spec: &ChainSpec,
+    ) -> Result<(), Error> {
+        let balance = self
+            .balances_mut()
+            .get_mut(validator_index)
+            .ok_or(Error::UnknownValidator(validator_index))?;
+        if *balance > spec.min_activation_balance {
+            let excess_balance = balance.safe_sub(spec.min_activation_balance)?;
+            *balance = spec.min_activation_balance;
+            self.pending_balance_deposits_mut()?
+                .push(PendingBalanceDeposit {
+                    index: validator_index as u64,
+                    amount: excess_balance,
+                })?;
+        }
+        Ok(())
+    }
+
+    pub fn queue_entire_balance_and_reset_validator(
+        &mut self,
+        validator_index: usize,
+        spec: &ChainSpec,
+    ) -> Result<(), Error> {
+        let balance = self
+            .balances_mut()
+            .get_mut(validator_index)
+            .ok_or(Error::UnknownValidator(validator_index))?;
+        let balance_copy = *balance;
+        *balance = 0_u64;
+
+        let validator = self
+            .validators_mut()
+            .get_mut(validator_index)
+            .ok_or(Error::UnknownValidator(validator_index))?;
+        validator.effective_balance = 0;
+        validator.activation_eligibility_epoch = spec.far_future_epoch;
+
+        self.pending_balance_deposits_mut()?
+            .push(PendingBalanceDeposit {
+                index: validator_index as u64,
+                amount: balance_copy,
+            })
+            .map_err(Into::into)
+    }
+
     #[allow(clippy::arithmetic_side_effects)]
     pub fn rebase_on(&mut self, base: &Self, spec: &ChainSpec) -> Result<(), Error> {
         // Required for macros (which use type-hints internally).
@@ -2147,10 +2258,17 @@ impl<E: EthSpec> BeaconState<E> {
     /// The number of fields of the `BeaconState` rounded up to the nearest power of two.
     ///
     /// This is relevant to tree-hashing of the `BeaconState`.
-    ///
-    /// We assume this value is stable across forks. This assumption is checked in the
-    /// `check_num_fields_pow2` test.
-    pub const NUM_FIELDS_POW2: usize = BeaconStateBellatrix::<E>::NUM_FIELDS.next_power_of_two();
+    pub fn num_fields_pow2(&self) -> usize {
+        let fork_name = self.fork_name_unchecked();
+        match fork_name {
+            ForkName::Base => BeaconStateBase::<E>::NUM_FIELDS.next_power_of_two(),
+            ForkName::Altair => BeaconStateAltair::<E>::NUM_FIELDS.next_power_of_two(),
+            ForkName::Bellatrix => BeaconStateBellatrix::<E>::NUM_FIELDS.next_power_of_two(),
+            ForkName::Capella => BeaconStateCapella::<E>::NUM_FIELDS.next_power_of_two(),
+            ForkName::Deneb => BeaconStateDeneb::<E>::NUM_FIELDS.next_power_of_two(),
+            ForkName::Electra => BeaconStateElectra::<E>::NUM_FIELDS.next_power_of_two(),
+        }
+    }
 
     /// Specialised deserialisation method that uses the `ChainSpec` as context.
     #[allow(clippy::arithmetic_side_effects)]
@@ -2211,7 +2329,7 @@ impl<E: EthSpec> BeaconState<E> {
                 // in the `BeaconState`:
                 // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#beaconstate
                 generalized_index
-                    .checked_sub(Self::NUM_FIELDS_POW2)
+                    .checked_sub(self.num_fields_pow2())
                     .ok_or(Error::IndexNotSupported(generalized_index))?
             }
             light_client_update::FINALIZED_ROOT_INDEX => {
@@ -2221,7 +2339,7 @@ impl<E: EthSpec> BeaconState<E> {
                 // Subtract off the internal nodes. Result should be 105/2 - 32 = 20 which matches
                 // position of `finalized_checkpoint` in `BeaconState`.
                 finalized_checkpoint_generalized_index
-                    .checked_sub(Self::NUM_FIELDS_POW2)
+                    .checked_sub(self.num_fields_pow2())
                     .ok_or(Error::IndexNotSupported(generalized_index))?
             }
             _ => return Err(Error::IndexNotSupported(generalized_index)),