merge queue: embarking unstable (5121d65) and [#5406 + #5397 + #4946] together

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -1347,11 +1347,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         (parent_root, slot, sync_aggregate): LightClientProducerEvent<T::EthSpec>,
     ) -> Result<(), Error> {
         self.light_client_server_cache.recompute_and_cache_updates(
-            &self.log,
             self.store.clone(),
             &parent_root,
             slot,
             &sync_aggregate,
+            &self.log,
+            &self.spec,
         )
     }
 
@@ -6635,13 +6636,17 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         &self,
         block_root: &Hash256,
     ) -> Result<Option<(LightClientBootstrap<T::EthSpec>, ForkName)>, Error> {
-        let Some((state_root, slot)) = self
-            .get_blinded_block(block_root)?
-            .map(|block| (block.state_root(), block.slot()))
-        else {
+        let handle = self
+            .task_executor
+            .handle()
+            .ok_or(BeaconChainError::RuntimeShutdown)?;
+
+        let Some(block) = handle.block_on(async { self.get_block(block_root).await })? else {
             return Ok(None);
         };
 
+        let (state_root, slot) = (block.state_root(), block.slot());
+
         let Some(mut state) = self.get_state(&state_root, Some(slot))? else {
             return Ok(None);
         };
@@ -6651,12 +6656,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .map_err(Error::InconsistentFork)?;
 
         match fork_name {
-            ForkName::Altair | ForkName::Merge => {
-                LightClientBootstrap::from_beacon_state(&mut state)
+            ForkName::Altair | ForkName::Merge | ForkName::Capella | ForkName::Deneb => {
+                LightClientBootstrap::from_beacon_state(&mut state, &block, &self.spec)
                     .map(|bootstrap| Some((bootstrap, fork_name)))
                     .map_err(Error::LightClientError)
             }
-            ForkName::Base | ForkName::Capella | ForkName::Deneb => Err(Error::UnsupportedFork),
+            ForkName::Base => Err(Error::UnsupportedFork),
         }
     }
 }

beacon_node/beacon_chain/src/errors.rs

@@ -250,6 +250,7 @@ easy_from_to!(StateAdvanceError, BeaconChainError);
 easy_from_to!(BlockReplayError, BeaconChainError);
 easy_from_to!(InconsistentFork, BeaconChainError);
 easy_from_to!(AvailabilityCheckError, BeaconChainError);
+easy_from_to!(LightClientError, BeaconChainError);
 
 #[derive(Debug)]
 pub enum BlockProductionError {

beacon_node/beacon_chain/src/light_client_finality_update_verification.rs

@@ -48,7 +48,7 @@ impl<T: BeaconChainTypes> VerifiedLightClientFinalityUpdate<T> {
         // verify that enough time has passed for the block to have been propagated
         let start_time = chain
             .slot_clock
-            .start_of(rcv_finality_update.signature_slot)
+            .start_of(*rcv_finality_update.signature_slot())
             .ok_or(Error::SigSlotStartIsNone)?;
         let one_third_slot_duration = Duration::new(chain.spec.seconds_per_slot / 3, 0);
         if seen_timestamp + chain.spec.maximum_gossip_clock_disparity()

beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs

@@ -52,7 +52,7 @@ impl<T: BeaconChainTypes> VerifiedLightClientOptimisticUpdate<T> {
         // verify that enough time has passed for the block to have been propagated
         let start_time = chain
             .slot_clock
-            .start_of(rcv_optimistic_update.signature_slot)
+            .start_of(*rcv_optimistic_update.signature_slot())
             .ok_or(Error::SigSlotStartIsNone)?;
         let one_third_slot_duration = Duration::new(chain.spec.seconds_per_slot / 3, 0);
         if seen_timestamp + chain.spec.maximum_gossip_clock_disparity()
@@ -65,10 +65,7 @@ impl<T: BeaconChainTypes> VerifiedLightClientOptimisticUpdate<T> {
         let head_block = &head.snapshot.beacon_block;
         // check if we can process the optimistic update immediately
         // otherwise queue
-        let canonical_root = rcv_optimistic_update
-            .attested_header
-            .beacon
-            .canonical_root();
+        let canonical_root = rcv_optimistic_update.get_canonical_root();
 
         if canonical_root != head_block.message().parent_root() {
             return Err(Error::UnknownBlockParentRoot(canonical_root));
@@ -84,7 +81,7 @@ impl<T: BeaconChainTypes> VerifiedLightClientOptimisticUpdate<T> {
             return Err(Error::InvalidLightClientOptimisticUpdate);
         }
 
-        let parent_root = rcv_optimistic_update.attested_header.beacon.parent_root;
+        let parent_root = rcv_optimistic_update.get_parent_root();
         Ok(Self {
             light_client_optimistic_update: rcv_optimistic_update,
             parent_root,

beacon_node/beacon_chain/src/light_client_server_cache.rs

@@ -8,7 +8,7 @@ use types::light_client_update::{FinalizedRootProofLen, FINALIZED_ROOT_INDEX};
 use types::non_zero_usize::new_non_zero_usize;
 use types::{
     BeaconBlockRef, BeaconState, ChainSpec, EthSpec, ForkName, Hash256, LightClientFinalityUpdate,
-    LightClientHeader, LightClientOptimisticUpdate, Slot, SyncAggregate,
+    LightClientOptimisticUpdate, Slot, SyncAggregate,
 };
 
 /// A prev block cache miss requires to re-generate the state of the post-parent block. Items in the
@@ -71,24 +71,26 @@ impl<T: BeaconChainTypes> LightClientServerCache<T> {
     /// results are cached either on disk or memory to be served via p2p and rest API
     pub fn recompute_and_cache_updates(
         &self,
-        log: &Logger,
         store: BeaconStore<T>,
         block_parent_root: &Hash256,
         block_slot: Slot,
         sync_aggregate: &SyncAggregate<T::EthSpec>,
+        log: &Logger,
+        chain_spec: &ChainSpec,
     ) -> Result<(), BeaconChainError> {
         let _timer =
             metrics::start_timer(&metrics::LIGHT_CLIENT_SERVER_CACHE_RECOMPUTE_UPDATES_TIMES);
 
         let signature_slot = block_slot;
         let attested_block_root = block_parent_root;
 
-        let attested_block = store.get_blinded_block(attested_block_root)?.ok_or(
-            BeaconChainError::DBInconsistent(format!(
-                "Block not available {:?}",
-                attested_block_root
-            )),
-        )?;
+        let attested_block =
+            store
+                .get_full_block(attested_block_root)?
+                .ok_or(BeaconChainError::DBInconsistent(format!(
+                    "Block not available {:?}",
+                    attested_block_root
+                )))?;
 
         let cached_parts = self.get_or_compute_prev_block_cache(
             store.clone(),
@@ -109,11 +111,12 @@ impl<T: BeaconChainTypes> LightClientServerCache<T> {
         };
         if is_latest_optimistic {
             // can create an optimistic update, that is more recent
-            *self.latest_optimistic_update.write() = Some(LightClientOptimisticUpdate {
-                attested_header: block_to_light_client_header(attested_block.message()),
-                sync_aggregate: sync_aggregate.clone(),
+            *self.latest_optimistic_update.write() = Some(LightClientOptimisticUpdate::new(
+                &attested_block,
+                sync_aggregate.clone(),
                 signature_slot,
-            });
+                chain_spec,
+            )?);
         };
 
         // Spec: Full nodes SHOULD provide the LightClientFinalityUpdate with the highest
@@ -127,17 +130,16 @@ impl<T: BeaconChainTypes> LightClientServerCache<T> {
         if is_latest_finality & !cached_parts.finalized_block_root.is_zero() {
             // Immediately after checkpoint sync the finalized block may not be available yet.
             if let Some(finalized_block) =
-                store.get_blinded_block(&cached_parts.finalized_block_root)?
+                store.get_full_block(&cached_parts.finalized_block_root)?
             {
-                *self.latest_finality_update.write() = Some(LightClientFinalityUpdate {
-                    // TODO: may want to cache this result from latest_optimistic_update if producing a
-                    // light_client header becomes expensive
-                    attested_header: block_to_light_client_header(attested_block.message()),
-                    finalized_header: block_to_light_client_header(finalized_block.message()),
-                    finality_branch: cached_parts.finality_branch.clone(),
-                    sync_aggregate: sync_aggregate.clone(),
+                *self.latest_finality_update.write() = Some(LightClientFinalityUpdate::new(
+                    &attested_block,
+                    &finalized_block,
+                    cached_parts.finality_branch.clone(),
+                    sync_aggregate.clone(),
                     signature_slot,
-                });
+                    chain_spec,
+                )?);
             } else {
                 debug!(
                     log,
@@ -214,7 +216,7 @@ impl LightClientCachedData {
     }
 }
 
-// Implements spec priorization rules:
+// Implements spec prioritization rules:
 // > Full nodes SHOULD provide the LightClientFinalityUpdate with the highest attested_header.beacon.slot (if multiple, highest signature_slot)
 //
 // ref: https://github.com/ethereum/consensus-specs/blob/113c58f9bf9c08867f6f5f633c4d98e0364d612a/specs/altair/light-client/full-node.md#create_light_client_finality_update
@@ -223,14 +225,15 @@ fn is_latest_finality_update<T: EthSpec>(
     attested_slot: Slot,
     signature_slot: Slot,
 ) -> bool {
-    if attested_slot > prev.attested_header.beacon.slot {
+    let prev_slot = prev.get_attested_header_slot();
+    if attested_slot > prev_slot {
         true
     } else {
-        attested_slot == prev.attested_header.beacon.slot && signature_slot > prev.signature_slot
+        attested_slot == prev_slot && signature_slot > *prev.signature_slot()
     }
 }
 
-// Implements spec priorization rules:
+// Implements spec prioritization rules:
 // > Full nodes SHOULD provide the LightClientOptimisticUpdate with the highest attested_header.beacon.slot (if multiple, highest signature_slot)
 //
 // ref: https://github.com/ethereum/consensus-specs/blob/113c58f9bf9c08867f6f5f633c4d98e0364d612a/specs/altair/light-client/full-node.md#create_light_client_optimistic_update
@@ -239,18 +242,10 @@ fn is_latest_optimistic_update<T: EthSpec>(
     attested_slot: Slot,
     signature_slot: Slot,
 ) -> bool {
-    if attested_slot > prev.attested_header.beacon.slot {
+    let prev_slot = prev.get_slot();
+    if attested_slot > prev_slot {
         true
     } else {
-        attested_slot == prev.attested_header.beacon.slot && signature_slot > prev.signature_slot
-    }
-}
-
-fn block_to_light_client_header<T: EthSpec>(
-    block: BeaconBlockRef<T, types::BlindedPayload<T>>,
-) -> LightClientHeader {
-    // TODO: make fork aware
-    LightClientHeader {
-        beacon: block.block_header(),
+        attested_slot == prev_slot && signature_slot > *prev.signature_slot()
     }
 }

beacon_node/beacon_processor/src/lib.rs

@@ -284,7 +284,7 @@ pub struct BeaconProcessorChannels<E: EthSpec> {
 impl<E: EthSpec> BeaconProcessorChannels<E> {
     pub fn new(config: &BeaconProcessorConfig) -> Self {
         let (beacon_processor_tx, beacon_processor_rx) =
-            mpsc::channel(config.max_scheduled_work_queue_len);
+            mpsc::channel(config.max_work_event_queue_len);
         let (work_reprocessing_tx, work_reprocessing_rx) =
             mpsc::channel(config.max_scheduled_work_queue_len);
 

beacon_node/execution_layer/src/engine_api/http.rs

@@ -71,23 +71,6 @@ pub static LIGHTHOUSE_CAPABILITIES: &[&str] = &[
     ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1,
 ];
 
-/// This is necessary because a user might run a capella-enabled version of
-/// lighthouse before they update to a capella-enabled execution engine.
-// TODO (mark): rip this out once we are post-capella on mainnet
-pub static PRE_CAPELLA_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities {
-    new_payload_v1: true,
-    new_payload_v2: false,
-    new_payload_v3: false,
-    forkchoice_updated_v1: true,
-    forkchoice_updated_v2: false,
-    forkchoice_updated_v3: false,
-    get_payload_bodies_by_hash_v1: false,
-    get_payload_bodies_by_range_v1: false,
-    get_payload_v1: true,
-    get_payload_v2: false,
-    get_payload_v3: false,
-};
-
 /// Contains methods to convert arbitrary bytes to an ETH2 deposit contract object.
 pub mod deposit_log {
     use ssz::Decode;
@@ -1012,38 +995,29 @@ impl HttpJsonRpc {
     pub async fn exchange_capabilities(&self) -> Result<EngineCapabilities, Error> {
         let params = json!([LIGHTHOUSE_CAPABILITIES]);
 
-        let response: Result<HashSet<String>, _> = self
+        let capabilities: HashSet<String> = self
             .rpc_request(
                 ENGINE_EXCHANGE_CAPABILITIES,
                 params,
                 ENGINE_EXCHANGE_CAPABILITIES_TIMEOUT * self.execution_timeout_multiplier,
             )
-            .await;
+            .await?;
 
-        match response {
-            // TODO (mark): rip this out once we are post capella on mainnet
-            Err(error) => match error {
-                Error::ServerMessage { code, message: _ } if code == METHOD_NOT_FOUND_CODE => {
-                    Ok(PRE_CAPELLA_ENGINE_CAPABILITIES)
-                }
-                _ => Err(error),
-            },
-            Ok(capabilities) => Ok(EngineCapabilities {
-                new_payload_v1: capabilities.contains(ENGINE_NEW_PAYLOAD_V1),
-                new_payload_v2: capabilities.contains(ENGINE_NEW_PAYLOAD_V2),
-                new_payload_v3: capabilities.contains(ENGINE_NEW_PAYLOAD_V3),
-                forkchoice_updated_v1: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V1),
-                forkchoice_updated_v2: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V2),
-                forkchoice_updated_v3: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V3),
-                get_payload_bodies_by_hash_v1: capabilities
-                    .contains(ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1),
-                get_payload_bodies_by_range_v1: capabilities
-                    .contains(ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1),
-                get_payload_v1: capabilities.contains(ENGINE_GET_PAYLOAD_V1),
-                get_payload_v2: capabilities.contains(ENGINE_GET_PAYLOAD_V2),
-                get_payload_v3: capabilities.contains(ENGINE_GET_PAYLOAD_V3),
-            }),
-        }
+        Ok(EngineCapabilities {
+            new_payload_v1: capabilities.contains(ENGINE_NEW_PAYLOAD_V1),
+            new_payload_v2: capabilities.contains(ENGINE_NEW_PAYLOAD_V2),
+            new_payload_v3: capabilities.contains(ENGINE_NEW_PAYLOAD_V3),
+            forkchoice_updated_v1: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V1),
+            forkchoice_updated_v2: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V2),
+            forkchoice_updated_v3: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V3),
+            get_payload_bodies_by_hash_v1: capabilities
+                .contains(ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1),
+            get_payload_bodies_by_range_v1: capabilities
+                .contains(ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1),
+            get_payload_v1: capabilities.contains(ENGINE_GET_PAYLOAD_V1),
+            get_payload_v2: capabilities.contains(ENGINE_GET_PAYLOAD_V2),
+            get_payload_v3: capabilities.contains(ENGINE_GET_PAYLOAD_V3),
+        })
     }
 
     pub async fn clear_exchange_capabilties_cache(&self) {

beacon_node/http_api/src/lib.rs

@@ -2337,7 +2337,7 @@ pub fn serve<T: BeaconChainTypes>(
 
                     let fork_name = chain
                         .spec
-                        .fork_name_at_slot::<T::EthSpec>(update.signature_slot);
+                        .fork_name_at_slot::<T::EthSpec>(*update.signature_slot());
                     match accept_header {
                         Some(api_types::Accept::Ssz) => Response::builder()
                             .status(200)
@@ -2384,7 +2384,7 @@ pub fn serve<T: BeaconChainTypes>(
 
                     let fork_name = chain
                         .spec
-                        .fork_name_at_slot::<T::EthSpec>(update.signature_slot);
+                        .fork_name_at_slot::<T::EthSpec>(*update.signature_slot());
                     match accept_header {
                         Some(api_types::Accept::Ssz) => Response::builder()
                             .status(200)

beacon_node/http_api/tests/tests.rs

@@ -1717,7 +1717,7 @@ impl ApiTester {
         };
 
         let expected = block.slot();
-        assert_eq!(result.header.beacon.slot, expected);
+        assert_eq!(result.get_slot(), expected);
 
         self
     }

beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs

@@ -590,9 +590,18 @@ fn handle_rpc_response<T: EthSpec>(
         SupportedProtocol::MetaDataV1 => Ok(Some(RPCResponse::MetaData(MetaData::V1(
             MetaDataV1::from_ssz_bytes(decoded_buffer)?,
         )))),
-        SupportedProtocol::LightClientBootstrapV1 => Ok(Some(RPCResponse::LightClientBootstrap(
-            LightClientBootstrap::from_ssz_bytes(decoded_buffer)?,
-        ))),
+        SupportedProtocol::LightClientBootstrapV1 => match fork_name {
+            Some(fork_name) => Ok(Some(RPCResponse::LightClientBootstrap(Arc::new(
+                LightClientBootstrap::from_ssz_bytes(decoded_buffer, fork_name)?,
+            )))),
+            None => Err(RPCError::ErrorResponse(
+                RPCResponseErrorCode::InvalidRequest,
+                format!(
+                    "No context bytes provided for {:?} response",
+                    versioned_protocol
+                ),
+            )),
+        },
         // MetaData V2 responses have no context bytes, so behave similarly to V1 responses
         SupportedProtocol::MetaDataV2 => Ok(Some(RPCResponse::MetaData(MetaData::V2(
             MetaDataV2::from_ssz_bytes(decoded_buffer)?,

beacon_node/lighthouse_network/src/rpc/methods.rs

@@ -388,7 +388,7 @@ pub enum RPCResponse<T: EthSpec> {
     BlobsByRange(Arc<BlobSidecar<T>>),
 
     /// A response to a get LIGHT_CLIENT_BOOTSTRAP request.
-    LightClientBootstrap(LightClientBootstrap<T>),
+    LightClientBootstrap(Arc<LightClientBootstrap<T>>),
 
     /// A response to a get BLOBS_BY_ROOT request.
     BlobsByRoot(Arc<BlobSidecar<T>>),
@@ -569,11 +569,7 @@ impl<T: EthSpec> std::fmt::Display for RPCResponse<T> {
             RPCResponse::Pong(ping) => write!(f, "Pong: {}", ping.data),
             RPCResponse::MetaData(metadata) => write!(f, "Metadata: {}", metadata.seq_number()),
             RPCResponse::LightClientBootstrap(bootstrap) => {
-                write!(
-                    f,
-                    "LightClientBootstrap Slot: {}",
-                    bootstrap.header.beacon.slot
-                )
+                write!(f, "LightClientBootstrap Slot: {}", bootstrap.get_slot())
             }
         }
     }